Skip to content

Find, verify, and analyze leaked credentials

more detectors

Dustin Decker5596025b0b64
diff --git a/pkg/detectors/abbysale/abbysale_test.go b/pkg/detectors/abbysale/abbysale_test.go
index 2ac4e119f..0a55d131a 100644
--- a/pkg/detectors/abbysale/abbysale_test.go
+++ b/pkg/detectors/abbysale/abbysale_test.go
@@ -16,7 +16,7 @@ import (
func TestAbbysale_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/abstract/abstract_test.go b/pkg/detectors/abstract/abstract_test.go
index c937303b0..c34e44674 100644
--- a/pkg/detectors/abstract/abstract_test.go
+++ b/pkg/detectors/abstract/abstract_test.go
@@ -16,7 +16,7 @@ import (
func TestAbstract_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/abuseipdb/abuseipdb_test.go b/pkg/detectors/abuseipdb/abuseipdb_test.go
index ff3bc5d48..9772b6ed5 100644
--- a/pkg/detectors/abuseipdb/abuseipdb_test.go
+++ b/pkg/detectors/abuseipdb/abuseipdb_test.go
@@ -16,7 +16,7 @@ import (
func TestAbuseIPDB_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/accuweather/accuweather_test.go b/pkg/detectors/accuweather/accuweather_test.go
index 269ffcd00..c12bcc231 100644
--- a/pkg/detectors/accuweather/accuweather_test.go
+++ b/pkg/detectors/accuweather/accuweather_test.go
@@ -16,7 +16,7 @@ import (
func TestAccuweather_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/adafruitio/adafruitio_test.go b/pkg/detectors/adafruitio/adafruitio_test.go
index 028f8dd10..4a119c575 100644
--- a/pkg/detectors/adafruitio/adafruitio_test.go
+++ b/pkg/detectors/adafruitio/adafruitio_test.go
@@ -16,7 +16,7 @@ import (
func TestAdafruitIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/adobeio/adobeio_test.go b/pkg/detectors/adobeio/adobeio_test.go
index 5309903ec..031a69e33 100644
--- a/pkg/detectors/adobeio/adobeio_test.go
+++ b/pkg/detectors/adobeio/adobeio_test.go
@@ -16,7 +16,7 @@ import (
func TestAdobeIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/adzuna/adzuna_test.go b/pkg/detectors/adzuna/adzuna_test.go
index 123c27c6f..0e2598191 100644
--- a/pkg/detectors/adzuna/adzuna_test.go
+++ b/pkg/detectors/adzuna/adzuna_test.go
@@ -16,7 +16,7 @@ import (
func TestAdzuna_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/agora/agora.go b/pkg/detectors/agora/agora.go
new file mode 100644
index 000000000..1c6054d99
--- /dev/null
+++ b/pkg/detectors/agora/agora.go
@@ -0,0 +1,84 @@
+package agora
+
+import (
+ "context"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"agora"}) + `\b([a-z0-9]{32})\b`)
+ secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"agora"}) + `\b([a-z0-9]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"agora"}
+}
+
+// FromData will find and optionally verify Agora secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+
+ resMatch := strings.TrimSpace(match[1])
+
+ for _, secret := range secretMatches {
+ if len(secret) != 2 {
+ continue
+ }
+ resSecret := strings.TrimSpace(secret[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Agora,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://api.agora.io/dev/v1/projects", nil)
+ req.SetBasicAuth(resSecret, resMatch)
+ res, err := client.Do(req)
+
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/agora/agora_test.go b/pkg/detectors/agora/agora_test.go
new file mode 100644
index 000000000..d558c4113
--- /dev/null
+++ b/pkg/detectors/agora/agora_test.go
@@ -0,0 +1,114 @@
+package agora
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestAgora_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("AGORA")
+ secret := testSecrets.MustGetField("AGORA_SECRET")
+ inactiveSecret := testSecrets.MustGetField("AGORA_SECRET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but not valid ", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go
index 6ce727e8d..f4240b80c 100644
--- a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go
+++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go
@@ -16,7 +16,7 @@ import (
func TestAirbrakeProjectKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go
index 965f0e25d..57b2fa40d 100644
--- a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go
+++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go
@@ -16,7 +16,7 @@ import (
func TestAirbrakeUserKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/airship/airship_test.go b/pkg/detectors/airship/airship_test.go
index 778d337fe..dbc66fbbc 100644
--- a/pkg/detectors/airship/airship_test.go
+++ b/pkg/detectors/airship/airship_test.go
@@ -16,7 +16,7 @@ import (
func TestAirship_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/airtableapikey/airtableapikey_test.go b/pkg/detectors/airtableapikey/airtableapikey_test.go
index 01030a1ad..85ea10403 100644
--- a/pkg/detectors/airtableapikey/airtableapikey_test.go
+++ b/pkg/detectors/airtableapikey/airtableapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestAirtableApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/airvisual/airvisual_test.go b/pkg/detectors/airvisual/airvisual_test.go
index 97a45bcc2..9ed306dcc 100644
--- a/pkg/detectors/airvisual/airvisual_test.go
+++ b/pkg/detectors/airvisual/airvisual_test.go
@@ -16,7 +16,7 @@ import (
func TestAirVisual_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/alconost/alconost_test.go b/pkg/detectors/alconost/alconost_test.go
index baa98acae..45dc7b3bc 100644
--- a/pkg/detectors/alconost/alconost_test.go
+++ b/pkg/detectors/alconost/alconost_test.go
@@ -16,7 +16,7 @@ import (
func TestAlconost_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/alegra/alegra_test.go b/pkg/detectors/alegra/alegra_test.go
index 5c73d054a..44a0ceaa3 100644
--- a/pkg/detectors/alegra/alegra_test.go
+++ b/pkg/detectors/alegra/alegra_test.go
@@ -16,7 +16,7 @@ import (
func TestAlegra_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/aletheiaapi/aletheiaapi_test.go b/pkg/detectors/aletheiaapi/aletheiaapi_test.go
index 7e1a8fb01..1526f203a 100644
--- a/pkg/detectors/aletheiaapi/aletheiaapi_test.go
+++ b/pkg/detectors/aletheiaapi/aletheiaapi_test.go
@@ -16,7 +16,7 @@ import (
func TestAletheiaApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey.go b/pkg/detectors/algoliaadminkey/algoliaadminkey.go
new file mode 100644
index 000000000..fd7c079ab
--- /dev/null
+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey.go
@@ -0,0 +1,81 @@
+package algoliaadminkey
+
+import (
+ "context"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([a-zA-Z0-9]{32})\b`)
+ idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"algolia"}) + `\b([A-Z0-9]{10})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"algolia"}
+}
+
+// FromData will find and optionally verify AlgoliaAdminKey secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+ for _, idMatch := range idMatches {
+ if len(idMatch) != 2 {
+ continue
+ }
+ resIdMatch := strings.TrimSpace(idMatch[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://"+resIdMatch+"-dsn.algolia.net/1/keys", nil)
+ req.Header.Add("X-Algolia-Application-Id", resIdMatch)
+ req.Header.Add("X-Algolia-API-Key", resMatch)
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+ }
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go
new file mode 100644
index 000000000..5fc7b25b3
--- /dev/null
+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go
@@ -0,0 +1,114 @@
+package algoliaadminkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestAlgoliaAdminKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALGOLIAADMINKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("ALGOLIAADMINKEY_INACTIVE")
+ id := testSecrets.MustGetField("ALGOLIAADMINKEY_APPID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AlgoliaAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AlgoliaAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/alibaba/alibaba.go b/pkg/detectors/alibaba/alibaba.go
new file mode 100644
index 000000000..3cd8a3e5e
--- /dev/null
+++ b/pkg/detectors/alibaba/alibaba.go
@@ -0,0 +1,84 @@
+package alibaba
+
+import (
+ "context"
+ "regexp"
+ "time"
+
+ "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs"
+ log "github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(`\b(LTAI[a-zA-Z0-9]{17,21})[\"' ;\s]*`)
+ secretPat = regexp.MustCompile(`\b([a-zA-Z0-9]{30})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"LTAI"}
+}
+
+// FromData will find and optionally verify Alibaba secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ for _, match := range matches {
+ //Plausible key pat found, look for secrets match
+ secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, secMatch := range secMatches {
+
+ if len(match) != 2 {
+ continue
+ }
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Raw: []byte(match[1]),
+ Redacted: match[1],
+ }
+
+ if verify {
+ ecsClient, err := ecs.NewClientWithAccessKey(
+ "us-east-1", // your region ID
+ match[1], // your AccessKey ID
+ secMatch[1]) // your AccessKey Secret
+ if err != nil {
+ log.WithError(err).Debug("error creating alibaba client, skipping")
+ continue
+ }
+ // Create an API request and set parameters
+ request := ecs.CreateDescribeInstancesRequest()
+ request.ConnectTimeout = (time.Duration(5) * time.Second)
+ request.Scheme = "https"
+ request.Domain = "ecs.aliyuncs.com"
+ // Initiate the request and handle exceptions
+ _, err = ecsClient.DescribeInstances(request)
+ if err != nil {
+ s.Verified = false
+
+ } else {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+ }
+ return
+}
diff --git a/pkg/detectors/alibaba/alibaba_test.go b/pkg/detectors/alibaba/alibaba_test.go
new file mode 100644
index 000000000..2413d86de
--- /dev/null
+++ b/pkg/detectors/alibaba/alibaba_test.go
@@ -0,0 +1,103 @@
+package alibaba
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestAlibaba_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALIBABA_SECRET")
+ secretInactive := testSecrets.MustGetField("ALIBABA_SECRET_INACTIVE")
+ id := testSecrets.MustGetField("ALIBABA_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("Alibaba keys are here: %s\n %s within", id, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: true,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("Alibaba keys are here: %s\n %s within", id, secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: false,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/alienvault/alienvault_test.go b/pkg/detectors/alienvault/alienvault_test.go
index c698a4583..d045210a1 100644
--- a/pkg/detectors/alienvault/alienvault_test.go
+++ b/pkg/detectors/alienvault/alienvault_test.go
@@ -16,7 +16,7 @@ import (
func TestAlienVault_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/allsports/allsports_test.go b/pkg/detectors/allsports/allsports_test.go
index 6c62ff83c..b2acfe937 100644
--- a/pkg/detectors/allsports/allsports_test.go
+++ b/pkg/detectors/allsports/allsports_test.go
@@ -16,7 +16,7 @@ import (
func TestAllsports_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/amadeus/amadeus_test.go b/pkg/detectors/amadeus/amadeus_test.go
index 63e389409..699d6c334 100644
--- a/pkg/detectors/amadeus/amadeus_test.go
+++ b/pkg/detectors/amadeus/amadeus_test.go
@@ -16,7 +16,7 @@ import (
func TestAmadeus_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ambee/ambee_test.go b/pkg/detectors/ambee/ambee_test.go
index 962070bd1..e5293dcd6 100644
--- a/pkg/detectors/ambee/ambee_test.go
+++ b/pkg/detectors/ambee/ambee_test.go
@@ -16,7 +16,7 @@ import (
func TestAmbee_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go
index da8560a4b..3dd7bcf4b 100644
--- a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go
+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestAmplitudeApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/anypoint/anypoint_test.go b/pkg/detectors/anypoint/anypoint_test.go
index 99a8ec51e..ca1e6b58d 100644
--- a/pkg/detectors/anypoint/anypoint_test.go
+++ b/pkg/detectors/anypoint/anypoint_test.go
@@ -16,7 +16,7 @@ import (
func TestAnypoint_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apacta/apacta_test.go b/pkg/detectors/apacta/apacta_test.go
index 5aadfd0f0..238bce55c 100644
--- a/pkg/detectors/apacta/apacta_test.go
+++ b/pkg/detectors/apacta/apacta_test.go
@@ -16,7 +16,7 @@ import (
func TestApacta_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/api2cart/api2cart_test.go b/pkg/detectors/api2cart/api2cart_test.go
index a1a58e1fb..36d894d75 100644
--- a/pkg/detectors/api2cart/api2cart_test.go
+++ b/pkg/detectors/api2cart/api2cart_test.go
@@ -16,7 +16,7 @@ import (
func TestApi2Cart_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apideck/apideck_test.go b/pkg/detectors/apideck/apideck_test.go
index 539a12221..48f67af43 100644
--- a/pkg/detectors/apideck/apideck_test.go
+++ b/pkg/detectors/apideck/apideck_test.go
@@ -16,7 +16,7 @@ import (
func TestApiDeck_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apiflash/apiflash_test.go b/pkg/detectors/apiflash/apiflash_test.go
index 8ce3c4b0c..f916c9cae 100644
--- a/pkg/detectors/apiflash/apiflash_test.go
+++ b/pkg/detectors/apiflash/apiflash_test.go
@@ -16,7 +16,7 @@ import (
func TestApiflash_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apifonica/apifonica_test.go b/pkg/detectors/apifonica/apifonica_test.go
index 9c798f358..bfc616e09 100644
--- a/pkg/detectors/apifonica/apifonica_test.go
+++ b/pkg/detectors/apifonica/apifonica_test.go
@@ -16,7 +16,7 @@ import (
func TestApifonica_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apify/apify_test.go b/pkg/detectors/apify/apify_test.go
index e4ff1b219..9d5ebecc2 100644
--- a/pkg/detectors/apify/apify_test.go
+++ b/pkg/detectors/apify/apify_test.go
@@ -16,7 +16,7 @@ import (
func TestApify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apimatic/apimatic_test.go b/pkg/detectors/apimatic/apimatic_test.go
index aa4e707f4..b2cb41c7a 100644
--- a/pkg/detectors/apimatic/apimatic_test.go
+++ b/pkg/detectors/apimatic/apimatic_test.go
@@ -16,7 +16,7 @@ import (
func TestAPIMatic_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apiscience/apiscience_test.go b/pkg/detectors/apiscience/apiscience_test.go
index 52709f350..91cdf6b7f 100644
--- a/pkg/detectors/apiscience/apiscience_test.go
+++ b/pkg/detectors/apiscience/apiscience_test.go
@@ -16,7 +16,7 @@ import (
func TestApiScience_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apollo/apollo_test.go b/pkg/detectors/apollo/apollo_test.go
index 832cfcf98..eefab358a 100644
--- a/pkg/detectors/apollo/apollo_test.go
+++ b/pkg/detectors/apollo/apollo_test.go
@@ -16,7 +16,7 @@ import (
func TestApollo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/appcues/appcues_test.go b/pkg/detectors/appcues/appcues_test.go
index 927d2dd5e..0083e9d68 100644
--- a/pkg/detectors/appcues/appcues_test.go
+++ b/pkg/detectors/appcues/appcues_test.go
@@ -16,7 +16,7 @@ import (
func TestAppcues_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/appfollow/appfollow_test.go b/pkg/detectors/appfollow/appfollow_test.go
index 8816f9fd1..b9e4fca6b 100644
--- a/pkg/detectors/appfollow/appfollow_test.go
+++ b/pkg/detectors/appfollow/appfollow_test.go
@@ -16,7 +16,7 @@ import (
func TestAppfollow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/appsynergy/appsynergy_test.go b/pkg/detectors/appsynergy/appsynergy_test.go
index 8105a54e4..5be9a53e3 100644
--- a/pkg/detectors/appsynergy/appsynergy_test.go
+++ b/pkg/detectors/appsynergy/appsynergy_test.go
@@ -16,7 +16,7 @@ import (
func TestAppSynergy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/apptivo/apptivo_test.go b/pkg/detectors/apptivo/apptivo_test.go
index 4965e5edb..78c4d6b90 100644
--- a/pkg/detectors/apptivo/apptivo_test.go
+++ b/pkg/detectors/apptivo/apptivo_test.go
@@ -16,7 +16,7 @@ import (
func TestApptivo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/artifactory/artifactory_test.go b/pkg/detectors/artifactory/artifactory_test.go
index db674bae1..f5cf1ee29 100644
--- a/pkg/detectors/artifactory/artifactory_test.go
+++ b/pkg/detectors/artifactory/artifactory_test.go
@@ -16,7 +16,7 @@ import (
func TestArtifactory_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/artsy/artsy_test.go b/pkg/detectors/artsy/artsy_test.go
index c84675b41..7fe3d077b 100644
--- a/pkg/detectors/artsy/artsy_test.go
+++ b/pkg/detectors/artsy/artsy_test.go
@@ -16,7 +16,7 @@ import (
func TestArtsy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/asanaoauth/asanaoauth_test.go b/pkg/detectors/asanaoauth/asanaoauth_test.go
index 3bb5def32..24850a13a 100644
--- a/pkg/detectors/asanaoauth/asanaoauth_test.go
+++ b/pkg/detectors/asanaoauth/asanaoauth_test.go
@@ -16,7 +16,7 @@ import (
func TestAsanaOauth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go
index 31d0bbb34..0cea9f33a 100644
--- a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go
+++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go
@@ -16,7 +16,7 @@ import (
func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/assemblyai/assemblyai_test.go b/pkg/detectors/assemblyai/assemblyai_test.go
index 4f43f7730..4c6072093 100644
--- a/pkg/detectors/assemblyai/assemblyai_test.go
+++ b/pkg/detectors/assemblyai/assemblyai_test.go
@@ -16,7 +16,7 @@ import (
func TestAssemblyai_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/audd/audd_test.go b/pkg/detectors/audd/audd_test.go
index 2b0d2eced..f9d23f685 100644
--- a/pkg/detectors/audd/audd_test.go
+++ b/pkg/detectors/audd/audd_test.go
@@ -16,7 +16,7 @@ import (
func TestAudd_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go
index 97586c913..0143cbdd7 100644
--- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go
+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go
@@ -16,7 +16,7 @@ import (
func TestAuth0ManagementApiToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/auth0oauth/auth0oauth_test.go b/pkg/detectors/auth0oauth/auth0oauth_test.go
index 87dfd6d4d..68fd96352 100644
--- a/pkg/detectors/auth0oauth/auth0oauth_test.go
+++ b/pkg/detectors/auth0oauth/auth0oauth_test.go
@@ -16,7 +16,7 @@ import (
func TestAuth0oauth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/autodesk/autodesk_test.go b/pkg/detectors/autodesk/autodesk_test.go
index 08c368c44..a14b4585e 100644
--- a/pkg/detectors/autodesk/autodesk_test.go
+++ b/pkg/detectors/autodesk/autodesk_test.go
@@ -16,7 +16,7 @@ import (
func TestAutodesk_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/autoklose/autoklose_test.go b/pkg/detectors/autoklose/autoklose_test.go
index 5afd2baeb..a1684ec79 100644
--- a/pkg/detectors/autoklose/autoklose_test.go
+++ b/pkg/detectors/autoklose/autoklose_test.go
@@ -16,7 +16,7 @@ import (
func TestAutoklose_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/autopilot/autopilot_test.go b/pkg/detectors/autopilot/autopilot_test.go
index b4af56675..ba92646d9 100644
--- a/pkg/detectors/autopilot/autopilot_test.go
+++ b/pkg/detectors/autopilot/autopilot_test.go
@@ -16,7 +16,7 @@ import (
func TestAutoPilot_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/aviationstack/aviationstack_test.go b/pkg/detectors/aviationstack/aviationstack_test.go
index ef6c4c4cb..b16c85a53 100644
--- a/pkg/detectors/aviationstack/aviationstack_test.go
+++ b/pkg/detectors/aviationstack/aviationstack_test.go
@@ -16,7 +16,7 @@ import (
func TestAviationStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go
new file mode 100644
index 000000000..c6590f3ba
--- /dev/null
+++ b/pkg/detectors/aws/aws.go
@@ -0,0 +1,96 @@
+package aws
+
+import (
+ "context"
+ "regexp"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/sts"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ // TODO: Support other key types https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
+ keyPat = regexp.MustCompile(`\b(AKIA[0-9A-Z]{16})\b`)
+ secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"AKI"}
+}
+
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
+ dataStr := string(data)
+ var results []detectors.Result
+
+ keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, keyMatch := range keyMatches {
+ if len(keyMatch) != 2 {
+ continue
+ }
+
+ key := strings.TrimSpace(keyMatch[1])
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AWS,
+ Raw: []byte(key),
+ Redacted: key,
+ }
+ // TODO: Remove possible matches if they verify positive.
+ if verify {
+ for _, secretMatch := range secretMatches {
+ if len(secretMatch) != 2 {
+ continue
+ }
+
+ secret := strings.TrimSpace(secretMatch[1])
+
+ result, err := callerIdentity(ctx, key, secret)
+ if err != nil {
+ // It also errors for signature mismatches on the client side before sending, and it's quite noisy.
+ continue
+ }
+ if result != nil && result.Account != nil {
+ s.Verified = true
+ break
+ }
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ if len(secretMatches) > 0 {
+ results = append(results, s)
+ }
+ }
+
+ return detectors.CleanResults(results), nil
+}
+
+func callerIdentity(ctx context.Context, key, secret string) (*sts.GetCallerIdentityOutput, error) {
+ svc := sts.New(sts.Options{
+ HTTPClient: common.SaneHttpClient(),
+ Logger: nil,
+ Region: "us-west-2",
+ Credentials: credentials.NewStaticCredentialsProvider(key, secret, ""),
+ })
+ result, err := svc.GetCallerIdentity(ctx, nil)
+ return result, err
+}
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go
new file mode 100644
index 000000000..0afbd1fdf
--- /dev/null
+++ b/pkg/detectors/aws/aws_test.go
@@ -0,0 +1,167 @@
+package aws
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/service/sts"
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestScanner_FromChunk(t *testing.T) {
+
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AWS_SECRET")
+ secretInactive := testSecrets.MustGetField("AWS_INACTIVE")
+ id := testSecrets.MustGetField("AWS")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ args args
+ wantSecrets []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "live key",
+ args: args{
+ ctx: ctx,
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within awsId %s", secret, id)),
+
+ verify: true,
+ },
+ wantSecrets: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: true,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "dead key",
+ args: args{
+ ctx: ctx,
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within awsId %s", secretInactive, id)),
+ verify: true,
+ },
+ wantSecrets: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ args: args{
+ ctx: ctx,
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ wantSecrets: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Scanner.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.wantSecrets); diff != "" {
+ t.Errorf("%s: Scanner.FromData() diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
+
+func Test_callerIdentity(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AWS_SECRET")
+ secretInactive := testSecrets.MustGetField("AWS_INACTIVE")
+ id := testSecrets.MustGetField("AWS")
+
+ type args struct {
+ key string
+ secret string
+ ctx context.Context
+ }
+ tests := []struct {
+ name string
+ args args
+ want *sts.GetCallerIdentityOutput
+ wantErr bool
+ }{
+ {
+ name: "invalid",
+ args: args{
+ key: id,
+ secret: secretInactive,
+ ctx: context.Background(),
+ },
+ wantErr: true,
+ },
+ {
+ name: "valid",
+ args: args{
+ key: id,
+ secret: secret,
+ ctx: context.Background(),
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := callerIdentity(tt.args.ctx, tt.args.key, tt.args.secret)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("callerIdentity() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/axonaut/axonaut_test.go b/pkg/detectors/axonaut/axonaut_test.go
index 1b9a54739..03eaaebeb 100644
--- a/pkg/detectors/axonaut/axonaut_test.go
+++ b/pkg/detectors/axonaut/axonaut_test.go
@@ -16,7 +16,7 @@ import (
func TestAxonaut_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/aylien/aylien_test.go b/pkg/detectors/aylien/aylien_test.go
index b4822aa0b..65f22c650 100644
--- a/pkg/detectors/aylien/aylien_test.go
+++ b/pkg/detectors/aylien/aylien_test.go
@@ -16,7 +16,7 @@ import (
func TestAylien_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ayrshare/ayrshare_test.go b/pkg/detectors/ayrshare/ayrshare_test.go
index dd682c1a6..6fd26ad8f 100644
--- a/pkg/detectors/ayrshare/ayrshare_test.go
+++ b/pkg/detectors/ayrshare/ayrshare_test.go
@@ -16,7 +16,7 @@ import (
func TestAyrshare_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/azure/azure.go b/pkg/detectors/azure/azure.go
new file mode 100644
index 000000000..097b8f2a1
--- /dev/null
+++ b/pkg/detectors/azure/azure.go
@@ -0,0 +1,87 @@
+package azure
+
+import (
+ "context"
+ "fmt"
+ "regexp"
+ "strings"
+
+ "github.com/Azure/go-autorest/autorest/azure/auth"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+func mustFmtPat(id, pat string) *regexp.Regexp {
+ combinedID := strings.ReplaceAll(id, "_", "") + "|" + id
+ return regexp.MustCompile(fmt.Sprintf(pat, combinedID))
+}
+
+var (
+ // TODO: Azure storage access keys and investigate other types of creds.
+
+ // Azure App Oauth
+ idPatFmt = `(?i)(%s).{0,20}([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})`
+ clientIDPat = mustFmtPat("client_id", idPatFmt)
+ tenantIDPat = mustFmtPat("tenant_id", idPatFmt)
+
+ // TODO: support old patterns
+ secretPatFmt = `(?i)(%s).{0,20}([a-z0-9_\.\-~]{34})`
+ clientSecretPat = mustFmtPat("client_secret", secretPatFmt)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"azure"}
+}
+
+// FromData will find and optionally verify Azure secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
+ for _, clientSecret := range clientSecretMatches {
+ tenantIDMatches := tenantIDPat.FindAllStringSubmatch(dataStr, -1)
+ for _, tenantID := range tenantIDMatches {
+ clientIDMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
+ for _, clientID := range clientIDMatches {
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Azure,
+ Raw: []byte(clientSecret[2]),
+ Redacted: clientID[2],
+ }
+
+ if verify {
+ cred := auth.NewClientCredentialsConfig(clientID[2], clientSecret[2], tenantID[2])
+ token, err := cred.ServicePrincipalToken()
+ if err != nil {
+ continue
+ }
+ err = token.Refresh()
+ if err == nil {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(s.Redacted, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+ }
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/azure/azure_test.go b/pkg/detectors/azure/azure_test.go
new file mode 100644
index 000000000..8a5d7207d
--- /dev/null
+++ b/pkg/detectors/azure/azure_test.go
@@ -0,0 +1,110 @@
+package azure
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestAzure_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_SECRET")
+ secretInactive := testSecrets.MustGetField("AZURE_INACTIVE")
+ id := testSecrets.MustGetField("AZURE_ID")
+ tenantId := testSecrets.MustGetField("AZURE_TENANT_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`
+ tenant_id=%s
+ client_id=%s
+ client_secret=%s
+ client_secret=%s
+ `, tenantId, id, secretInactive, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Azure,
+ Redacted: id,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`
+ tenant_id=%s
+ client_id=%s
+ client_secret=%s
+ `, tenantId, id, secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Azure,
+ Redacted: id,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/bannerbear/bannerbear_test.go b/pkg/detectors/bannerbear/bannerbear_test.go
index 9f9c5a022..625c8ee23 100644
--- a/pkg/detectors/bannerbear/bannerbear_test.go
+++ b/pkg/detectors/bannerbear/bannerbear_test.go
@@ -16,7 +16,7 @@ import (
func TestBannerbear_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/baremetrics/baremetrics_test.go b/pkg/detectors/baremetrics/baremetrics_test.go
index bf1654fb7..1397b86dc 100644
--- a/pkg/detectors/baremetrics/baremetrics_test.go
+++ b/pkg/detectors/baremetrics/baremetrics_test.go
@@ -16,7 +16,7 @@ import (
func TestBaremetrics_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/baseapiio/baseapiio_test.go b/pkg/detectors/baseapiio/baseapiio_test.go
index fb988ac8d..591d167c9 100644
--- a/pkg/detectors/baseapiio/baseapiio_test.go
+++ b/pkg/detectors/baseapiio/baseapiio_test.go
@@ -16,7 +16,7 @@ import (
func TestBaseApiIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/beamer/beamer_test.go b/pkg/detectors/beamer/beamer_test.go
index 81e5eed6d..18617452c 100644
--- a/pkg/detectors/beamer/beamer_test.go
+++ b/pkg/detectors/beamer/beamer_test.go
@@ -16,7 +16,7 @@ import (
func TestBeamer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/beebole/beebole_test.go b/pkg/detectors/beebole/beebole_test.go
index 6de959947..0c6132142 100644
--- a/pkg/detectors/beebole/beebole_test.go
+++ b/pkg/detectors/beebole/beebole_test.go
@@ -16,7 +16,7 @@ import (
func TestBeebole_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/besttime/besttime_test.go b/pkg/detectors/besttime/besttime_test.go
index 847db3fc1..9e5eae6d7 100644
--- a/pkg/detectors/besttime/besttime_test.go
+++ b/pkg/detectors/besttime/besttime_test.go
@@ -16,7 +16,7 @@ import (
func TestBesttime_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/billomat/billomat_test.go b/pkg/detectors/billomat/billomat_test.go
index 9193922ac..421bcd0e2 100644
--- a/pkg/detectors/billomat/billomat_test.go
+++ b/pkg/detectors/billomat/billomat_test.go
@@ -16,7 +16,7 @@ import (
func TestBillomat_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bitbar/bitbar_test.go b/pkg/detectors/bitbar/bitbar_test.go
index 7fd158e7a..50b631ee3 100644
--- a/pkg/detectors/bitbar/bitbar_test.go
+++ b/pkg/detectors/bitbar/bitbar_test.go
@@ -16,7 +16,7 @@ import (
func TestBitbar_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go
index 4dafb76a3..ccd97104c 100644
--- a/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go
+++ b/pkg/detectors/bitcoinaverage/bitcoinaverage_test.go
@@ -16,7 +16,7 @@ import (
func TestBitcoinAverage_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bitfinex/bitfinex_test.go b/pkg/detectors/bitfinex/bitfinex_test.go
index 13d53af61..1d8bc5b6f 100644
--- a/pkg/detectors/bitfinex/bitfinex_test.go
+++ b/pkg/detectors/bitfinex/bitfinex_test.go
@@ -16,7 +16,7 @@ import (
func TestBitfinex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go
index 140087708..6e46c8fe1 100644
--- a/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go
+++ b/pkg/detectors/bitlyaccesstoken/bitlyaccesstoken_test.go
@@ -16,7 +16,7 @@ import (
func TestBitLyAccessToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bitmex/bitmex_test.go b/pkg/detectors/bitmex/bitmex_test.go
index 2a5b1f54b..ee06131d0 100644
--- a/pkg/detectors/bitmex/bitmex_test.go
+++ b/pkg/detectors/bitmex/bitmex_test.go
@@ -16,7 +16,7 @@ import (
func TestBitmex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/blablabus/blablabus_test.go b/pkg/detectors/blablabus/blablabus_test.go
index 3724355be..3eca42cfc 100644
--- a/pkg/detectors/blablabus/blablabus_test.go
+++ b/pkg/detectors/blablabus/blablabus_test.go
@@ -16,7 +16,7 @@ import (
func TestBlablabus_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/blazemeter/blazemeter_test.go b/pkg/detectors/blazemeter/blazemeter_test.go
index 5cec03e47..87c48b2a8 100644
--- a/pkg/detectors/blazemeter/blazemeter_test.go
+++ b/pkg/detectors/blazemeter/blazemeter_test.go
@@ -16,7 +16,7 @@ import (
func TestBlazemeter_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/blitapp/blitapp_test.go b/pkg/detectors/blitapp/blitapp_test.go
index 3583ef810..9a8c8cff8 100644
--- a/pkg/detectors/blitapp/blitapp_test.go
+++ b/pkg/detectors/blitapp/blitapp_test.go
@@ -16,7 +16,7 @@ import (
func TestBlitApp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/blogger/blogger_test.go b/pkg/detectors/blogger/blogger_test.go
index fecd80aeb..241ee722b 100644
--- a/pkg/detectors/blogger/blogger_test.go
+++ b/pkg/detectors/blogger/blogger_test.go
@@ -16,7 +16,7 @@ import (
func TestBlogger_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bombbomb/bombbomb_test.go b/pkg/detectors/bombbomb/bombbomb_test.go
index 8fb198a5c..ee4def006 100644
--- a/pkg/detectors/bombbomb/bombbomb_test.go
+++ b/pkg/detectors/bombbomb/bombbomb_test.go
@@ -16,7 +16,7 @@ import (
func TestBombBomb_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/brandfetch/brandfetch_test.go b/pkg/detectors/brandfetch/brandfetch_test.go
index 3e87ac047..8c524bc41 100644
--- a/pkg/detectors/brandfetch/brandfetch_test.go
+++ b/pkg/detectors/brandfetch/brandfetch_test.go
@@ -16,7 +16,7 @@ import (
func TestBrandfetch_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/browshot/browshot_test.go b/pkg/detectors/browshot/browshot_test.go
index 207216d40..4e60b5392 100644
--- a/pkg/detectors/browshot/browshot_test.go
+++ b/pkg/detectors/browshot/browshot_test.go
@@ -16,7 +16,7 @@ import (
func TestBrowshot_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/buddyns/buddyns_test.go b/pkg/detectors/buddyns/buddyns_test.go
index 085c566bc..ca6a187d8 100644
--- a/pkg/detectors/buddyns/buddyns_test.go
+++ b/pkg/detectors/buddyns/buddyns_test.go
@@ -16,7 +16,7 @@ import (
func TestBuddyns_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bugherd/bugherd_test.go b/pkg/detectors/bugherd/bugherd_test.go
index edb2d9d12..e5a41aded 100644
--- a/pkg/detectors/bugherd/bugherd_test.go
+++ b/pkg/detectors/bugherd/bugherd_test.go
@@ -16,7 +16,7 @@ import (
func TestBugherd_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bugsnag/bugsnag_test.go b/pkg/detectors/bugsnag/bugsnag_test.go
index 89891fa07..f5043aaff 100644
--- a/pkg/detectors/bugsnag/bugsnag_test.go
+++ b/pkg/detectors/bugsnag/bugsnag_test.go
@@ -16,7 +16,7 @@ import (
func TestBugsnag_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/buildkite/buildkite_test.go b/pkg/detectors/buildkite/buildkite_test.go
index 7fb826623..0a951607a 100644
--- a/pkg/detectors/buildkite/buildkite_test.go
+++ b/pkg/detectors/buildkite/buildkite_test.go
@@ -16,7 +16,7 @@ import (
func TestBuildkite_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/bulbul/bulbul_test.go b/pkg/detectors/bulbul/bulbul_test.go
index cd0bf8f83..92af8317d 100644
--- a/pkg/detectors/bulbul/bulbul_test.go
+++ b/pkg/detectors/bulbul/bulbul_test.go
@@ -16,7 +16,7 @@ import (
func TestBulbul_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/buttercms/buttercms_test.go b/pkg/detectors/buttercms/buttercms_test.go
index c0d4060b4..1ee550292 100644
--- a/pkg/detectors/buttercms/buttercms_test.go
+++ b/pkg/detectors/buttercms/buttercms_test.go
@@ -16,7 +16,7 @@ import (
func TestButterCMS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/caflou/caflou_test.go b/pkg/detectors/caflou/caflou_test.go
index 5c8659f93..fd415b925 100644
--- a/pkg/detectors/caflou/caflou_test.go
+++ b/pkg/detectors/caflou/caflou_test.go
@@ -16,7 +16,7 @@ import (
func TestCaflou_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/calendarific/calendarific_test.go b/pkg/detectors/calendarific/calendarific_test.go
index f3bed174c..27cd9d4e5 100644
--- a/pkg/detectors/calendarific/calendarific_test.go
+++ b/pkg/detectors/calendarific/calendarific_test.go
@@ -16,7 +16,7 @@ import (
func TestCalendarific_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/calendlyapikey/calendlyapikey_test.go b/pkg/detectors/calendlyapikey/calendlyapikey_test.go
index 5babe5e55..eb6c95932 100644
--- a/pkg/detectors/calendlyapikey/calendlyapikey_test.go
+++ b/pkg/detectors/calendlyapikey/calendlyapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestCalendlyApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/calorieninja/calorieninja_test.go b/pkg/detectors/calorieninja/calorieninja_test.go
index 3b7c2733c..e58e4fd27 100644
--- a/pkg/detectors/calorieninja/calorieninja_test.go
+++ b/pkg/detectors/calorieninja/calorieninja_test.go
@@ -16,7 +16,7 @@ import (
func TestCalorieninja_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/campayn/campayn_test.go b/pkg/detectors/campayn/campayn_test.go
index 0a43f0987..a92199053 100644
--- a/pkg/detectors/campayn/campayn_test.go
+++ b/pkg/detectors/campayn/campayn_test.go
@@ -16,7 +16,7 @@ import (
func TestCampayn_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cannyio/cannyio_test.go b/pkg/detectors/cannyio/cannyio_test.go
index 2688f7cc6..e42ff851c 100644
--- a/pkg/detectors/cannyio/cannyio_test.go
+++ b/pkg/detectors/cannyio/cannyio_test.go
@@ -16,7 +16,7 @@ import (
func TestCannyIo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/capsulecrm/capsulecrm_test.go b/pkg/detectors/capsulecrm/capsulecrm_test.go
index 134c3aa6f..e36edb7c2 100644
--- a/pkg/detectors/capsulecrm/capsulecrm_test.go
+++ b/pkg/detectors/capsulecrm/capsulecrm_test.go
@@ -16,7 +16,7 @@ import (
func TestCapsuleCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/carboninterface/carboninterface_test.go b/pkg/detectors/carboninterface/carboninterface_test.go
index 3b117130f..19515acdb 100644
--- a/pkg/detectors/carboninterface/carboninterface_test.go
+++ b/pkg/detectors/carboninterface/carboninterface_test.go
@@ -16,7 +16,7 @@ import (
func TestCarbonInterface_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cashboard/cashboard_test.go b/pkg/detectors/cashboard/cashboard_test.go
index aa2f4375e..6272bec34 100644
--- a/pkg/detectors/cashboard/cashboard_test.go
+++ b/pkg/detectors/cashboard/cashboard_test.go
@@ -16,7 +16,7 @@ import (
func TestCashboard_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/censys/censys_test.go b/pkg/detectors/censys/censys_test.go
index 7a9d2f6ce..e6adfe563 100644
--- a/pkg/detectors/censys/censys_test.go
+++ b/pkg/detectors/censys/censys_test.go
@@ -16,7 +16,7 @@ import (
func TestCensys_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/centralstationcrm/centralstationcrm_test.go b/pkg/detectors/centralstationcrm/centralstationcrm_test.go
index e04ebf527..fdc48dd5d 100644
--- a/pkg/detectors/centralstationcrm/centralstationcrm_test.go
+++ b/pkg/detectors/centralstationcrm/centralstationcrm_test.go
@@ -16,7 +16,7 @@ import (
func TestCentralStationCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/chatbot/chatbot_test.go b/pkg/detectors/chatbot/chatbot_test.go
index ee902c42c..a27c07bd0 100644
--- a/pkg/detectors/chatbot/chatbot_test.go
+++ b/pkg/detectors/chatbot/chatbot_test.go
@@ -16,7 +16,7 @@ import (
func TestChatbot_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/checio/checio_test.go b/pkg/detectors/checio/checio_test.go
index 649f61c74..832b67876 100644
--- a/pkg/detectors/checio/checio_test.go
+++ b/pkg/detectors/checio/checio_test.go
@@ -16,7 +16,7 @@ import (
func TestChecIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/checklyhq/checklyhq_test.go b/pkg/detectors/checklyhq/checklyhq_test.go
index 52511ac12..cbb674591 100644
--- a/pkg/detectors/checklyhq/checklyhq_test.go
+++ b/pkg/detectors/checklyhq/checklyhq_test.go
@@ -16,7 +16,7 @@ import (
func TestChecklyHQ_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/checkout/checkout_test.go b/pkg/detectors/checkout/checkout_test.go
index 567e3e91e..77c2a29e0 100644
--- a/pkg/detectors/checkout/checkout_test.go
+++ b/pkg/detectors/checkout/checkout_test.go
@@ -16,7 +16,7 @@ import (
func TestCheckout_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cicero/cicero_test.go b/pkg/detectors/cicero/cicero_test.go
index 3fdbb601a..277bdd895 100644
--- a/pkg/detectors/cicero/cicero_test.go
+++ b/pkg/detectors/cicero/cicero_test.go
@@ -16,7 +16,7 @@ import (
func TestCicero_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/circleci/circleci.go b/pkg/detectors/circleci/circleci.go
new file mode 100644
index 000000000..5fb9da4ad
--- /dev/null
+++ b/pkg/detectors/circleci/circleci.go
@@ -0,0 +1,69 @@
+package circleci
+
+import (
+ "context"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"circle"}) + `([a-fA-F0-9]{40})`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"circle"}
+}
+
+// FromData will find and optionally verify Circle secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+
+ token := match[1]
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Circle,
+ Raw: []byte(token),
+ }
+
+ if verify {
+ client := common.RetryableHttpClient()
+ // https://circleci.com/docs/api/#authentication
+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://circleci.com/api/v2/me", nil)
+ req.Header.Add("Accept", "application/json;")
+ req.Header.Add("Circle-Token", token)
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ }
+ if res != nil && res.StatusCode >= 200 && res.StatusCode < 300 {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/circleci/circleci_test.go b/pkg/detectors/circleci/circleci_test.go
new file mode 100644
index 000000000..573506c4a
--- /dev/null
+++ b/pkg/detectors/circleci/circleci_test.go
@@ -0,0 +1,115 @@
+package circleci
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestCircleCI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CIRCLECI")
+ secretInactive := testSecrets.MustGetField("CIRCLECI_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Circle,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Circle,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CircleCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CircleCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clarifai/clarifai_test.go b/pkg/detectors/clarifai/clarifai_test.go
index 72f1bdf62..cadf6d5da 100644
--- a/pkg/detectors/clarifai/clarifai_test.go
+++ b/pkg/detectors/clarifai/clarifai_test.go
@@ -16,7 +16,7 @@ import (
func TestClarifai_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clearbit/clearbit_test.go b/pkg/detectors/clearbit/clearbit_test.go
index 5fb4c3548..350ab194b 100644
--- a/pkg/detectors/clearbit/clearbit_test.go
+++ b/pkg/detectors/clearbit/clearbit_test.go
@@ -16,7 +16,7 @@ import (
func TestClearbit_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clickhelp/clickhelp_test.go b/pkg/detectors/clickhelp/clickhelp_test.go
index 6eb6ceec5..4a61b43b9 100644
--- a/pkg/detectors/clickhelp/clickhelp_test.go
+++ b/pkg/detectors/clickhelp/clickhelp_test.go
@@ -16,7 +16,7 @@ import (
func TestClickhelp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clicksendsms/clicksendsms_test.go b/pkg/detectors/clicksendsms/clicksendsms_test.go
index eadd04145..18ed1cf24 100644
--- a/pkg/detectors/clicksendsms/clicksendsms_test.go
+++ b/pkg/detectors/clicksendsms/clicksendsms_test.go
@@ -16,7 +16,7 @@ import (
func TestClickSendsms_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go
index 1ea4bc3a7..ff2a64a4c 100644
--- a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go
+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go
@@ -16,7 +16,7 @@ import (
func TestClickupPersonalToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clinchpad/clinchpad_test.go b/pkg/detectors/clinchpad/clinchpad_test.go
index 4abf634f3..ccfdc8c34 100644
--- a/pkg/detectors/clinchpad/clinchpad_test.go
+++ b/pkg/detectors/clinchpad/clinchpad_test.go
@@ -16,7 +16,7 @@ import (
func TestClinchpad_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clockify/clockify_test.go b/pkg/detectors/clockify/clockify_test.go
index 01e9e336a..16187fad3 100644
--- a/pkg/detectors/clockify/clockify_test.go
+++ b/pkg/detectors/clockify/clockify_test.go
@@ -16,7 +16,7 @@ import (
func TestClockify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clockworksms/clockworksms_test.go b/pkg/detectors/clockworksms/clockworksms_test.go
index 106f9eca1..b53d0a051 100644
--- a/pkg/detectors/clockworksms/clockworksms_test.go
+++ b/pkg/detectors/clockworksms/clockworksms_test.go
@@ -16,7 +16,7 @@ import (
func TestClockworksms_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/closecrm/close_test.go b/pkg/detectors/closecrm/close_test.go
index cb081abd7..41cb42002 100644
--- a/pkg/detectors/closecrm/close_test.go
+++ b/pkg/detectors/closecrm/close_test.go
@@ -16,7 +16,7 @@ import (
func TestClose_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloudelements/cloudelements_test.go b/pkg/detectors/cloudelements/cloudelements_test.go
index 755691ca4..24fa7daa0 100644
--- a/pkg/detectors/cloudelements/cloudelements_test.go
+++ b/pkg/detectors/cloudelements/cloudelements_test.go
@@ -16,7 +16,7 @@ import (
func TestCloudElements_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go
index d7d43faa2..4ab326639 100644
--- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go
+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go
@@ -16,7 +16,7 @@ import (
func TestCloudflareApiToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go
index 207ce0440..799ada693 100644
--- a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go
+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go
@@ -16,7 +16,7 @@ import (
func TestCloudflareCaKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go
index 8434dacb4..f6e7ece54 100644
--- a/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go
+++ b/pkg/detectors/cloudflareglobalapikey/cloudflareglobalapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestCloudflareGlobalApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloudmersive/cloudmersive_test.go b/pkg/detectors/cloudmersive/cloudmersive_test.go
index c82b64479..58da6f1e5 100644
--- a/pkg/detectors/cloudmersive/cloudmersive_test.go
+++ b/pkg/detectors/cloudmersive/cloudmersive_test.go
@@ -16,7 +16,7 @@ import (
func TestCloudmersive_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloudplan/cloudplan_test.go b/pkg/detectors/cloudplan/cloudplan_test.go
index 2ffd68a96..55cbe4383 100644
--- a/pkg/detectors/cloudplan/cloudplan_test.go
+++ b/pkg/detectors/cloudplan/cloudplan_test.go
@@ -16,7 +16,7 @@ import (
func TestCloudplan_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cloverly/cloverly_test.go b/pkg/detectors/cloverly/cloverly_test.go
index e49a98042..65272dea6 100644
--- a/pkg/detectors/cloverly/cloverly_test.go
+++ b/pkg/detectors/cloverly/cloverly_test.go
@@ -16,7 +16,7 @@ import (
func TestCloverly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/clustdoc/clustdoc_test.go b/pkg/detectors/clustdoc/clustdoc_test.go
index 0aa2e8e77..83cc00876 100644
--- a/pkg/detectors/clustdoc/clustdoc_test.go
+++ b/pkg/detectors/clustdoc/clustdoc_test.go
@@ -16,7 +16,7 @@ import (
func TestClustDoc_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/codacy/codacy_test.go b/pkg/detectors/codacy/codacy_test.go
index 59a37429c..a8fe6dcdb 100644
--- a/pkg/detectors/codacy/codacy_test.go
+++ b/pkg/detectors/codacy/codacy_test.go
@@ -16,7 +16,7 @@ import (
func TestCodacy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/coinapi/coinapi_test.go b/pkg/detectors/coinapi/coinapi_test.go
index 0ee75e690..e30cbd142 100644
--- a/pkg/detectors/coinapi/coinapi_test.go
+++ b/pkg/detectors/coinapi/coinapi_test.go
@@ -16,7 +16,7 @@ import (
func TestCoinApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/coinbase/coinbase_test.go b/pkg/detectors/coinbase/coinbase_test.go
index c8d77af63..289b81ee2 100644
--- a/pkg/detectors/coinbase/coinbase_test.go
+++ b/pkg/detectors/coinbase/coinbase_test.go
@@ -16,7 +16,7 @@ package coinbase
// func TestCoinbase_FromChunk(t *testing.T) {
// ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
// defer cancel()
-// testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+// testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
// if err != nil {
// t.Fatalf("could not get test secrets from GCP: %s", err)
// }
diff --git a/pkg/detectors/coinlayer/coinlayer_test.go b/pkg/detectors/coinlayer/coinlayer_test.go
index d0d832b24..7c83a87e6 100644
--- a/pkg/detectors/coinlayer/coinlayer_test.go
+++ b/pkg/detectors/coinlayer/coinlayer_test.go
@@ -16,7 +16,7 @@ import (
func TestCoinlayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/coinlib/coinlib_test.go b/pkg/detectors/coinlib/coinlib_test.go
index bcd64cb54..d6b15f389 100644
--- a/pkg/detectors/coinlib/coinlib_test.go
+++ b/pkg/detectors/coinlib/coinlib_test.go
@@ -16,7 +16,7 @@ import (
func TestCoinlib_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/commercejs/commercejs_test.go b/pkg/detectors/commercejs/commercejs_test.go
index 3950336a1..50f9e1e93 100644
--- a/pkg/detectors/commercejs/commercejs_test.go
+++ b/pkg/detectors/commercejs/commercejs_test.go
@@ -16,7 +16,7 @@ import (
func TestCommerceJS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/commodities/commodities_test.go b/pkg/detectors/commodities/commodities_test.go
index f0021a333..3f83e518a 100644
--- a/pkg/detectors/commodities/commodities_test.go
+++ b/pkg/detectors/commodities/commodities_test.go
@@ -16,7 +16,7 @@ import (
func TestCommodities_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/companyhub/companyhub_test.go b/pkg/detectors/companyhub/companyhub_test.go
index db3c32361..1588ae321 100644
--- a/pkg/detectors/companyhub/companyhub_test.go
+++ b/pkg/detectors/companyhub/companyhub_test.go
@@ -16,7 +16,7 @@ import (
func TestCompanyHub_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/confluent/confluent_test.go b/pkg/detectors/confluent/confluent_test.go
index f55467ab1..a5a634acf 100644
--- a/pkg/detectors/confluent/confluent_test.go
+++ b/pkg/detectors/confluent/confluent_test.go
@@ -16,7 +16,7 @@ import (
func TestConfluent_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go
index a06fdb186..f03585a26 100644
--- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go
+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go
@@ -16,7 +16,7 @@ import (
func TestContentfulPersonalAccessToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/convertkit/convertkit_test.go b/pkg/detectors/convertkit/convertkit_test.go
index cd1cb4c49..4405c2bc3 100644
--- a/pkg/detectors/convertkit/convertkit_test.go
+++ b/pkg/detectors/convertkit/convertkit_test.go
@@ -16,7 +16,7 @@ import (
func TestConvertkit_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/copper/copper_test.go b/pkg/detectors/copper/copper_test.go
index 00210b860..b3363304d 100644
--- a/pkg/detectors/copper/copper_test.go
+++ b/pkg/detectors/copper/copper_test.go
@@ -16,7 +16,7 @@ import (
func TestCopper_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/countrylayer/countrylayer_test.go b/pkg/detectors/countrylayer/countrylayer_test.go
index de26723f7..3641e69b4 100644
--- a/pkg/detectors/countrylayer/countrylayer_test.go
+++ b/pkg/detectors/countrylayer/countrylayer_test.go
@@ -16,7 +16,7 @@ import (
func TestCountryLayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/courier/courier_test.go b/pkg/detectors/courier/courier_test.go
index c210d49b7..4f130ab26 100644
--- a/pkg/detectors/courier/courier_test.go
+++ b/pkg/detectors/courier/courier_test.go
@@ -16,7 +16,7 @@ import (
func TestCourier_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/coveralls/coveralls_test.go b/pkg/detectors/coveralls/coveralls_test.go
index ae165f359..69da7122c 100644
--- a/pkg/detectors/coveralls/coveralls_test.go
+++ b/pkg/detectors/coveralls/coveralls_test.go
@@ -16,7 +16,7 @@ import (
func TestCoveralls_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/crowdin/crowdin_test.go b/pkg/detectors/crowdin/crowdin_test.go
index 24c2d4122..a1516452f 100644
--- a/pkg/detectors/crowdin/crowdin_test.go
+++ b/pkg/detectors/crowdin/crowdin_test.go
@@ -16,7 +16,7 @@ import (
func TestCrowdin_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/cryptocompare/cryptocompare_test.go b/pkg/detectors/cryptocompare/cryptocompare_test.go
index 01dc4d024..4459935f3 100644
--- a/pkg/detectors/cryptocompare/cryptocompare_test.go
+++ b/pkg/detectors/cryptocompare/cryptocompare_test.go
@@ -16,7 +16,7 @@ import (
func TestCryptoCompare_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/currencycloud/currencycloud_test.go b/pkg/detectors/currencycloud/currencycloud_test.go
index e2c40b056..710e2b596 100644
--- a/pkg/detectors/currencycloud/currencycloud_test.go
+++ b/pkg/detectors/currencycloud/currencycloud_test.go
@@ -16,7 +16,7 @@ import (
func TestCurrencycloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/currencyfreaks/currencyfreaks_test.go b/pkg/detectors/currencyfreaks/currencyfreaks_test.go
index 17d0b0449..8583abe0c 100644
--- a/pkg/detectors/currencyfreaks/currencyfreaks_test.go
+++ b/pkg/detectors/currencyfreaks/currencyfreaks_test.go
@@ -16,7 +16,7 @@ import (
func TestCurrencyfreaks_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/currencylayer/currencylayer_test.go b/pkg/detectors/currencylayer/currencylayer_test.go
index 19771d2e5..13706c143 100644
--- a/pkg/detectors/currencylayer/currencylayer_test.go
+++ b/pkg/detectors/currencylayer/currencylayer_test.go
@@ -16,7 +16,7 @@ import (
func TestCurrencylayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/currencyscoop/currencyscoop_test.go b/pkg/detectors/currencyscoop/currencyscoop_test.go
index 8c3d02514..d18a68bf4 100644
--- a/pkg/detectors/currencyscoop/currencyscoop_test.go
+++ b/pkg/detectors/currencyscoop/currencyscoop_test.go
@@ -16,7 +16,7 @@ import (
func TestCurrencyscoop_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/currentsapi/currentsapi_test.go b/pkg/detectors/currentsapi/currentsapi_test.go
index 403a788f4..63c607fe5 100644
--- a/pkg/detectors/currentsapi/currentsapi_test.go
+++ b/pkg/detectors/currentsapi/currentsapi_test.go
@@ -16,7 +16,7 @@ import (
func TestCurrentsAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/customerguru/customerguru_test.go b/pkg/detectors/customerguru/customerguru_test.go
index 1a9334efe..05cc9f319 100644
--- a/pkg/detectors/customerguru/customerguru_test.go
+++ b/pkg/detectors/customerguru/customerguru_test.go
@@ -16,7 +16,7 @@ import (
func TestCustomerGuru_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/customerio/customerio_test.go b/pkg/detectors/customerio/customerio_test.go
index acaa3a557..e9531ae0d 100644
--- a/pkg/detectors/customerio/customerio_test.go
+++ b/pkg/detectors/customerio/customerio_test.go
@@ -16,7 +16,7 @@ import (
func TestCustomerIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/d7network/d7network_test.go b/pkg/detectors/d7network/d7network_test.go
index f578454be..abfa04b48 100644
--- a/pkg/detectors/d7network/d7network_test.go
+++ b/pkg/detectors/d7network/d7network_test.go
@@ -16,7 +16,7 @@ import (
func TestD7Network_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dailyco/dailyco_test.go b/pkg/detectors/dailyco/dailyco_test.go
index dd81bde4b..b20a656e6 100644
--- a/pkg/detectors/dailyco/dailyco_test.go
+++ b/pkg/detectors/dailyco/dailyco_test.go
@@ -16,7 +16,7 @@ import (
func TestDailyCO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dandelion/dandelion_test.go b/pkg/detectors/dandelion/dandelion_test.go
index 6da14a140..013b8d05f 100644
--- a/pkg/detectors/dandelion/dandelion_test.go
+++ b/pkg/detectors/dandelion/dandelion_test.go
@@ -16,7 +16,7 @@ import (
func TestDandelion_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/datadogtoken/datadogtoken_test.go b/pkg/detectors/datadogtoken/datadogtoken_test.go
index 4fc6ba9b7..4bd76c87a 100644
--- a/pkg/detectors/datadogtoken/datadogtoken_test.go
+++ b/pkg/detectors/datadogtoken/datadogtoken_test.go
@@ -16,7 +16,7 @@ import (
func TestDatadogToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/datafire/datafire_test.go b/pkg/detectors/datafire/datafire_test.go
index afa3cd417..354e93a24 100644
--- a/pkg/detectors/datafire/datafire_test.go
+++ b/pkg/detectors/datafire/datafire_test.go
@@ -16,7 +16,7 @@ import (
func TestDataFire_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/datagov/datagov.go b/pkg/detectors/datagov/datagov.go
new file mode 100644
index 000000000..e45930ec4
--- /dev/null
+++ b/pkg/detectors/datagov/datagov.go
@@ -0,0 +1,73 @@
+package datagov
+
+import (
+ "context"
+ "fmt"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"data.gov"}) + `\b([a-zA-Z0-9]{40})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"data.gov"}
+}
+
+// FromData will find and optionally verify DataGov secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_DataGov,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", fmt.Sprintf("https://api.ers.usda.gov/data/arms/state?api_key=%s", resMatch), nil)
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/datagov/datagov_test.go b/pkg/detectors/datagov/datagov_test.go
new file mode 100644
index 000000000..7fe5d6b8e
--- /dev/null
+++ b/pkg/detectors/datagov/datagov_test.go
@@ -0,0 +1,113 @@
+package datagov
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestDataGov_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("DATAGOV")
+ inactiveSecret := testSecrets.MustGetField("DATAGOV_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a data.gov secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_DataGov,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a data.gov secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_DataGov,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("DataGov.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("DataGov.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/debounce/debounce_test.go b/pkg/detectors/debounce/debounce_test.go
index 166535b67..d0d03f438 100644
--- a/pkg/detectors/debounce/debounce_test.go
+++ b/pkg/detectors/debounce/debounce_test.go
@@ -16,7 +16,7 @@ import (
func TestDebounce_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/deepai/deepai_test.go b/pkg/detectors/deepai/deepai_test.go
index 2ccb662bf..e83d5ea21 100644
--- a/pkg/detectors/deepai/deepai_test.go
+++ b/pkg/detectors/deepai/deepai_test.go
@@ -16,7 +16,7 @@ import (
func TestDeepAI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/deepgram/deepgram_test.go b/pkg/detectors/deepgram/deepgram_test.go
index 9370b1847..21bba55bd 100644
--- a/pkg/detectors/deepgram/deepgram_test.go
+++ b/pkg/detectors/deepgram/deepgram_test.go
@@ -16,7 +16,7 @@ import (
func TestDeepgram_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/delighted/delighted_test.go b/pkg/detectors/delighted/delighted_test.go
index 66262fe54..d72756a13 100644
--- a/pkg/detectors/delighted/delighted_test.go
+++ b/pkg/detectors/delighted/delighted_test.go
@@ -16,7 +16,7 @@ import (
func TestDelighted_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/deputy/deputy_test.go b/pkg/detectors/deputy/deputy_test.go
index 268a490b8..fe9881d91 100644
--- a/pkg/detectors/deputy/deputy_test.go
+++ b/pkg/detectors/deputy/deputy_test.go
@@ -16,7 +16,7 @@ import (
func TestDeputy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/detectlanguage/detectlanguage_test.go b/pkg/detectors/detectlanguage/detectlanguage_test.go
index 81e429a22..78470889e 100644
--- a/pkg/detectors/detectlanguage/detectlanguage_test.go
+++ b/pkg/detectors/detectlanguage/detectlanguage_test.go
@@ -16,7 +16,7 @@ import (
func TestDetectLanguage_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go
index 5cc1fb7f4..cd3097ce5 100644
--- a/pkg/detectors/detectors.go
+++ b/pkg/detectors/detectors.go
@@ -2,6 +2,11 @@ package detectors
import (
"context"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "unicode"
"github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
@@ -61,11 +66,11 @@ func CleanResults(results []Result) []Result {
return results
}
- var cleaned = make([]Result, 0)
+ var cleaned = make(map[string]Result, 0)
for _, s := range results {
if s.Verified {
- cleaned = append(cleaned, s)
+ cleaned[s.Redacted] = s
}
}
@@ -73,5 +78,53 @@ func CleanResults(results []Result) []Result {
return results[:1]
}
- return cleaned
+ results = results[:0]
+ for _, r := range cleaned {
+ results = append(results, r)
+ }
+
+ return results
+}
+
+// Prefix regex ensures that at least one of the given keywords is within
+// 20 characters of the capturing group that follows.
+// This can help prevent false positives.
+func PrefixRegex(keywords []string) string {
+ pre := `(?i)(?:`
+ middle := strings.Join(keywords, "|")
+ post := `).{0,40}`
+ return pre + middle + post
+}
+
+//KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
+//Golang doesnt support regex lookaheads, so must be done in seperate calls.
+//TODO improve checks. Shannon entropy did not work well.
+func KeyIsRandom(key string) bool {
+ for _, ch := range key {
+ if unicode.IsDigit(ch) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func MustGetBenchmarkData() map[string][]byte {
+ _, filename, _, _ := runtime.Caller(0)
+ dir := filepath.Dir(filename)
+ small := make([]byte, 0)
+ medium, err := os.ReadFile(filepath.Join(dir, "detectors.go"))
+ if err != nil {
+ panic(err)
+ }
+ big := make([]byte, 0)
+ for i := 0; i < 25; i++ {
+ big = append(big, medium...)
+ }
+
+ return map[string][]byte{
+ "small": small,
+ "medium": medium,
+ "big": big,
+ }
}
diff --git a/pkg/detectors/dfuse/dfuse_test.go b/pkg/detectors/dfuse/dfuse_test.go
index fdce7e7f6..82481742d 100644
--- a/pkg/detectors/dfuse/dfuse_test.go
+++ b/pkg/detectors/dfuse/dfuse_test.go
@@ -16,7 +16,7 @@ import (
func TestDfuse_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go b/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go
index febacde97..1acb821fc 100644
--- a/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go
+++ b/pkg/detectors/digitaloceantoken/digitaloceantoken_test.go
@@ -16,7 +16,7 @@ import (
func TestDigitalOceanToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/discordbottoken/discordbottoken_test.go b/pkg/detectors/discordbottoken/discordbottoken_test.go
index aadd33e4f..3c0a8c7a2 100644
--- a/pkg/detectors/discordbottoken/discordbottoken_test.go
+++ b/pkg/detectors/discordbottoken/discordbottoken_test.go
@@ -16,7 +16,7 @@ import (
func TestDiscordBotToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/discordwebhook/discordwebhook_test.go b/pkg/detectors/discordwebhook/discordwebhook_test.go
index 806354ccb..c6ce7a1f0 100644
--- a/pkg/detectors/discordwebhook/discordwebhook_test.go
+++ b/pkg/detectors/discordwebhook/discordwebhook_test.go
@@ -16,7 +16,7 @@ import (
func TestDiscordWebhook_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ditto/ditto_test.go b/pkg/detectors/ditto/ditto_test.go
index accc3037e..13f8ee7e8 100644
--- a/pkg/detectors/ditto/ditto_test.go
+++ b/pkg/detectors/ditto/ditto_test.go
@@ -16,7 +16,7 @@ import (
func TestDitto_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dnscheck/dnscheck_test.go b/pkg/detectors/dnscheck/dnscheck_test.go
index c94b540e7..6da82e114 100644
--- a/pkg/detectors/dnscheck/dnscheck_test.go
+++ b/pkg/detectors/dnscheck/dnscheck_test.go
@@ -16,7 +16,7 @@ import (
func TestDnscheck_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/documo/documo_test.go b/pkg/detectors/documo/documo_test.go
index 89bc2498c..41a3c7442 100644
--- a/pkg/detectors/documo/documo_test.go
+++ b/pkg/detectors/documo/documo_test.go
@@ -16,7 +16,7 @@ import (
func TestDocumo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/doppler/doppler_test.go b/pkg/detectors/doppler/doppler_test.go
index 981280dfe..a51b4baf9 100644
--- a/pkg/detectors/doppler/doppler_test.go
+++ b/pkg/detectors/doppler/doppler_test.go
@@ -16,7 +16,7 @@ import (
func TestDoppler_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dotmailer/dotmailer_test.go b/pkg/detectors/dotmailer/dotmailer_test.go
index 569b44f3b..17df982e5 100644
--- a/pkg/detectors/dotmailer/dotmailer_test.go
+++ b/pkg/detectors/dotmailer/dotmailer_test.go
@@ -16,7 +16,7 @@ import (
func TestDotmailer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dovico/dovico_test.go b/pkg/detectors/dovico/dovico_test.go
index e1c17de82..81145dcfd 100644
--- a/pkg/detectors/dovico/dovico_test.go
+++ b/pkg/detectors/dovico/dovico_test.go
@@ -16,7 +16,7 @@ import (
func TestDovico_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dronahq/dronahq_test.go b/pkg/detectors/dronahq/dronahq_test.go
index 2768d50e6..e1a185005 100644
--- a/pkg/detectors/dronahq/dronahq_test.go
+++ b/pkg/detectors/dronahq/dronahq_test.go
@@ -16,7 +16,7 @@ import (
func TestDronaHQ_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/droneci/droneci_test.go b/pkg/detectors/droneci/droneci_test.go
index f78e5ef19..ea6fc8db9 100644
--- a/pkg/detectors/droneci/droneci_test.go
+++ b/pkg/detectors/droneci/droneci_test.go
@@ -16,7 +16,7 @@ import (
func TestDroneCI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dropbox/dropbox.go b/pkg/detectors/dropbox/dropbox.go
new file mode 100644
index 000000000..d3b5f6c00
--- /dev/null
+++ b/pkg/detectors/dropbox/dropbox.go
@@ -0,0 +1,76 @@
+package dropbox
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(`\b(sl\.[A-Za-z0-9\-\_]{130,140})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"sl."}
+}
+
+// FromData will find and optionally verify Dropbox secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Dropbox,
+ Raw: []byte(match[1]),
+ }
+
+ if verify {
+
+ baseURL := "https://api.dropboxapi.com/2/users/get_current_account"
+
+ client := common.RetryableHttpClient()
+
+ req, _ := http.NewRequest("POST", baseURL, nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ // 200 means good key for get current user
+ // 400 is bad (malformed)
+ // 403 bad scope
+ if res.StatusCode == http.StatusOK {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/dropbox/dropbox_test.go b/pkg/detectors/dropbox/dropbox_test.go
new file mode 100644
index 000000000..7d7060e9a
--- /dev/null
+++ b/pkg/detectors/dropbox/dropbox_test.go
@@ -0,0 +1,108 @@
+package dropbox
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestDropbox_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("DROPBOX")
+ secretInactive := testSecrets.MustGetField("DROPBOX_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a dropbox secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Dropbox,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a dropbox secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Dropbox,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Dropbox.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Dropbox.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/dwolla/dwolla_test.go b/pkg/detectors/dwolla/dwolla_test.go
index 033b2378e..cf3e2c996 100644
--- a/pkg/detectors/dwolla/dwolla_test.go
+++ b/pkg/detectors/dwolla/dwolla_test.go
@@ -16,7 +16,7 @@ import (
func TestDwolla_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dynalist/dynalist_test.go b/pkg/detectors/dynalist/dynalist_test.go
index 9aa303ad5..445bb537d 100644
--- a/pkg/detectors/dynalist/dynalist_test.go
+++ b/pkg/detectors/dynalist/dynalist_test.go
@@ -16,7 +16,7 @@ import (
func TestDynalist_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/dyspatch/dyspatch_test.go b/pkg/detectors/dyspatch/dyspatch_test.go
index 138813ceb..bba032d84 100644
--- a/pkg/detectors/dyspatch/dyspatch_test.go
+++ b/pkg/detectors/dyspatch/dyspatch_test.go
@@ -16,7 +16,7 @@ import (
func TestDyspatch_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go b/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go
index 9d715dbf8..7ab396558 100644
--- a/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go
+++ b/pkg/detectors/eagleeyenetworks/eagleeyenetworks_test.go
@@ -16,7 +16,7 @@ import (
func TestEagleEyeNetworks_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/easyinsight/easyinsight_test.go b/pkg/detectors/easyinsight/easyinsight_test.go
index 31130b871..3a8d7cc53 100644
--- a/pkg/detectors/easyinsight/easyinsight_test.go
+++ b/pkg/detectors/easyinsight/easyinsight_test.go
@@ -16,7 +16,7 @@ import (
func TestEasyInsight_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/edamam/edamam_test.go b/pkg/detectors/edamam/edamam_test.go
index 5bac8bc70..9fc200a3c 100644
--- a/pkg/detectors/edamam/edamam_test.go
+++ b/pkg/detectors/edamam/edamam_test.go
@@ -16,7 +16,7 @@ import (
func TestEdamam_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/edenai/edenai_test.go b/pkg/detectors/edenai/edenai_test.go
index ca3b52e0d..054aeee86 100644
--- a/pkg/detectors/edenai/edenai_test.go
+++ b/pkg/detectors/edenai/edenai_test.go
@@ -16,7 +16,7 @@ import (
func TestEdenAI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/elasticemail/elasticemail_test.go b/pkg/detectors/elasticemail/elasticemail_test.go
index 4f3749658..2148b81c1 100644
--- a/pkg/detectors/elasticemail/elasticemail_test.go
+++ b/pkg/detectors/elasticemail/elasticemail_test.go
@@ -16,7 +16,7 @@ import (
func TestElasticEmail_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/enigma/enigma_test.go b/pkg/detectors/enigma/enigma_test.go
index 460e25b2d..c7bd4014d 100644
--- a/pkg/detectors/enigma/enigma_test.go
+++ b/pkg/detectors/enigma/enigma_test.go
@@ -16,7 +16,7 @@ import (
func TestEnigma_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ethplorer/ethplorer_test.go b/pkg/detectors/ethplorer/ethplorer_test.go
index 2fff1ad26..048ae1789 100644
--- a/pkg/detectors/ethplorer/ethplorer_test.go
+++ b/pkg/detectors/ethplorer/ethplorer_test.go
@@ -16,7 +16,7 @@ import (
func TestEthplorer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/etsyapikey/etsyapikey_test.go b/pkg/detectors/etsyapikey/etsyapikey_test.go
index fdc964dc7..afc8396fc 100644
--- a/pkg/detectors/etsyapikey/etsyapikey_test.go
+++ b/pkg/detectors/etsyapikey/etsyapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestEtsyApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/everhour/everhour_test.go b/pkg/detectors/everhour/everhour_test.go
index 508be44e2..76c33c0b4 100644
--- a/pkg/detectors/everhour/everhour_test.go
+++ b/pkg/detectors/everhour/everhour_test.go
@@ -16,7 +16,7 @@ import (
func TestEverhour_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/exchangerateapi/exchangerateapi_test.go b/pkg/detectors/exchangerateapi/exchangerateapi_test.go
index 1b4af2022..010fe6905 100644
--- a/pkg/detectors/exchangerateapi/exchangerateapi_test.go
+++ b/pkg/detectors/exchangerateapi/exchangerateapi_test.go
@@ -16,7 +16,7 @@ import (
func TestExchangeRateAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go b/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go
index 238a497fa..ece1ceff5 100644
--- a/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go
+++ b/pkg/detectors/exchangeratesapi/exchangeratesapi_test.go
@@ -16,7 +16,7 @@ import (
func TestExchangeRatesAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/facebookoauth/facebookoauth_test.go b/pkg/detectors/facebookoauth/facebookoauth_test.go
index f836ce641..1a43759bf 100644
--- a/pkg/detectors/facebookoauth/facebookoauth_test.go
+++ b/pkg/detectors/facebookoauth/facebookoauth_test.go
@@ -16,7 +16,7 @@ import (
func TestFacebookOAuth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/faceplusplus/faceplusplus_test.go b/pkg/detectors/faceplusplus/faceplusplus_test.go
index ab087db52..03527f1a4 100644
--- a/pkg/detectors/faceplusplus/faceplusplus_test.go
+++ b/pkg/detectors/faceplusplus/faceplusplus_test.go
@@ -16,7 +16,7 @@ import (
func TestFaceplusplus_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fakejson/fakejson_test.go b/pkg/detectors/fakejson/fakejson_test.go
index 1e30d27cf..04fe4bccc 100644
--- a/pkg/detectors/fakejson/fakejson_test.go
+++ b/pkg/detectors/fakejson/fakejson_test.go
@@ -16,7 +16,7 @@ import (
func TestFakeJSON_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.go
new file mode 100644
index 000000000..b1ea5de67
--- /dev/null
+++ b/pkg/detectors/falsepositives.go
@@ -0,0 +1,92 @@
+package detectors
+
+import (
+ _ "embed"
+ "strings"
+ "unicode"
+)
+
+var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample"}
+
+type FalsePositive string
+
+//go:embed "badlist.txt"
+var badList []byte
+
+//go:embed "words.txt"
+var wordList []byte
+
+//go:embed "programmingbooks.txt"
+var programmingBookWords []byte
+
+type Wordlists struct {
+ wordList []string
+ badList []string
+ programmingBookWords []string
+}
+
+var FalsePositiveWordlists = Wordlists{
+ wordList: bytesToCleanWordList(wordList),
+ badList: bytesToCleanWordList(badList),
+ programmingBookWords: bytesToCleanWordList(programmingBookWords),
+}
+
+//IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
+//Currently that includes: No number, english word in key, or matches common example pattens.
+//Only the secret key material should be passed into this function
+func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
+
+ for _, fp := range falsePositives {
+ if strings.Contains(strings.ToLower(match), string(fp)) {
+ return true
+ }
+ }
+
+ if wordCheck {
+ // check against common substring badlist
+ if hasDictWord(FalsePositiveWordlists.badList, match) {
+ return true
+ }
+
+ // check for dictionary word substrings
+ if hasDictWord(FalsePositiveWordlists.wordList, match) {
+ return true
+ }
+
+ // check for programming book token substrings
+ if hasDictWord(FalsePositiveWordlists.programmingBookWords, match) {
+ return true
+ }
+ }
+ return false
+}
+
+func hasDictWord(wordList []string, token string) bool {
+ lower := strings.ToLower(token)
+ for _, word := range wordList {
+ if strings.Contains(lower, word) {
+ return true
+ }
+ }
+ return false
+}
+
+func HasDigit(key string) bool {
+ for _, ch := range key {
+ if unicode.IsDigit(ch) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func bytesToCleanWordList(data []byte) []string {
+ words := []string{}
+ for _, word := range strings.Split(string(data), "\n") {
+ if strings.TrimSpace(word) != "" {
+ words = append(words, strings.TrimSpace(strings.ToLower(word)))
+ }
+ }
+ return words
+}
diff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.go
new file mode 100644
index 000000000..2375aeca0
--- /dev/null
+++ b/pkg/detectors/falsepositives_test.go
@@ -0,0 +1,39 @@
+package detectors
+
+import "testing"
+
+func TestIsFalsePositive(t *testing.T) {
+ type args struct {
+ match string
+ falsePositives []FalsePositive
+ }
+ tests := []struct {
+ name string
+ args args
+ want bool
+ }{
+ {
+ name: "fp",
+ args: args{
+ match: "example",
+ falsePositives: DefaultFalsePositives,
+ },
+ want: true,
+ },
+ {
+ name: "not fp",
+ args: args{
+ match: "notafp123",
+ falsePositives: DefaultFalsePositives,
+ },
+ want: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := IsKnownFalsePositive(tt.args.match, tt.args.falsePositives, false); got != tt.want {
+ t.Errorf("IsKnownFalsePositive() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/fastforex/fastforex_test.go b/pkg/detectors/fastforex/fastforex_test.go
index e85aa4c6d..ef5d9e32d 100644
--- a/pkg/detectors/fastforex/fastforex_test.go
+++ b/pkg/detectors/fastforex/fastforex_test.go
@@ -16,7 +16,7 @@ import (
func TestFastForex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go
index 1f40893d3..f0f72750b 100644
--- a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go
+++ b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go
@@ -16,7 +16,7 @@ import (
func TestFastlyPersonalToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/feedier/feedier_test.go b/pkg/detectors/feedier/feedier_test.go
index 5f56f59ad..e66c72fed 100644
--- a/pkg/detectors/feedier/feedier_test.go
+++ b/pkg/detectors/feedier/feedier_test.go
@@ -16,7 +16,7 @@ import (
func TestFeedier_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fetchrss/fetchrss_test.go b/pkg/detectors/fetchrss/fetchrss_test.go
index 054b618fc..ff200feee 100644
--- a/pkg/detectors/fetchrss/fetchrss_test.go
+++ b/pkg/detectors/fetchrss/fetchrss_test.go
@@ -16,7 +16,7 @@ import (
func TestFetchrss_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go
index eee3e5e85..7f52ce4d8 100644
--- a/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go
+++ b/pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken_test.go
@@ -16,7 +16,7 @@ import (
func TestFigmaPersonalAccessToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fileio/fileio_test.go b/pkg/detectors/fileio/fileio_test.go
index b30fbb097..99ed368ea 100644
--- a/pkg/detectors/fileio/fileio_test.go
+++ b/pkg/detectors/fileio/fileio_test.go
@@ -16,7 +16,7 @@ import (
func TestFileIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/finage/finage_test.go b/pkg/detectors/finage/finage_test.go
index 24607feea..63f17f52d 100644
--- a/pkg/detectors/finage/finage_test.go
+++ b/pkg/detectors/finage/finage_test.go
@@ -16,7 +16,7 @@ import (
func TestFinage_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go b/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go
index 8b2d9a9df..183377b44 100644
--- a/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go
+++ b/pkg/detectors/financialmodelingprep/financialmodelingprep_test.go
@@ -16,7 +16,7 @@ import (
func TestFinancialModelingPrep_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/findl/findl_test.go b/pkg/detectors/findl/findl_test.go
index eb343f60e..b67ed0356 100644
--- a/pkg/detectors/findl/findl_test.go
+++ b/pkg/detectors/findl/findl_test.go
@@ -16,7 +16,7 @@ import (
func TestFindl_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/finnhub/finnhub_test.go b/pkg/detectors/finnhub/finnhub_test.go
index 9bd415a03..c5d211350 100644
--- a/pkg/detectors/finnhub/finnhub_test.go
+++ b/pkg/detectors/finnhub/finnhub_test.go
@@ -16,7 +16,7 @@ import (
func TestFinnhub_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fixerio/fixerio_test.go b/pkg/detectors/fixerio/fixerio_test.go
index 097a59c88..bfaa9c6b8 100644
--- a/pkg/detectors/fixerio/fixerio_test.go
+++ b/pkg/detectors/fixerio/fixerio_test.go
@@ -16,7 +16,7 @@ import (
func TestFixerIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/flatio/flatio_test.go b/pkg/detectors/flatio/flatio_test.go
index 0684dd875..6d900adcf 100644
--- a/pkg/detectors/flatio/flatio_test.go
+++ b/pkg/detectors/flatio/flatio_test.go
@@ -16,7 +16,7 @@ import (
func TestFlatIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fleetbase/fleetbase_test.go b/pkg/detectors/fleetbase/fleetbase_test.go
index 1c8da32e7..be301bc67 100644
--- a/pkg/detectors/fleetbase/fleetbase_test.go
+++ b/pkg/detectors/fleetbase/fleetbase_test.go
@@ -16,7 +16,7 @@ import (
func TestFleetbase_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/flickr/flickr_test.go b/pkg/detectors/flickr/flickr_test.go
index 96f9fdd44..31b41c99b 100644
--- a/pkg/detectors/flickr/flickr_test.go
+++ b/pkg/detectors/flickr/flickr_test.go
@@ -16,7 +16,7 @@ import (
func TestFlickr_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/flightapi/flightapi_test.go b/pkg/detectors/flightapi/flightapi_test.go
index 48fc8dbf8..c4b98e10d 100644
--- a/pkg/detectors/flightapi/flightapi_test.go
+++ b/pkg/detectors/flightapi/flightapi_test.go
@@ -16,7 +16,7 @@ import (
func TestFlightApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/float/float_test.go b/pkg/detectors/float/float_test.go
index da89df290..101eefd89 100644
--- a/pkg/detectors/float/float_test.go
+++ b/pkg/detectors/float/float_test.go
@@ -16,7 +16,7 @@ import (
func TestFloat_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/flowflu/flowflu_test.go b/pkg/detectors/flowflu/flowflu_test.go
index b565a87b1..7f7e2c938 100644
--- a/pkg/detectors/flowflu/flowflu_test.go
+++ b/pkg/detectors/flowflu/flowflu_test.go
@@ -16,7 +16,7 @@ import (
func TestFlowFlu_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/flutterwave/flutterwave_test.go b/pkg/detectors/flutterwave/flutterwave_test.go
index e6b571e4d..a09b96239 100644
--- a/pkg/detectors/flutterwave/flutterwave_test.go
+++ b/pkg/detectors/flutterwave/flutterwave_test.go
@@ -16,7 +16,7 @@ import (
func TestFlutterwave_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fmfw/fmfw_test.go b/pkg/detectors/fmfw/fmfw_test.go
index 648e78893..73c18b051 100644
--- a/pkg/detectors/fmfw/fmfw_test.go
+++ b/pkg/detectors/fmfw/fmfw_test.go
@@ -16,7 +16,7 @@ import (
func TestFmfw_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/foursquare/foursquare_test.go b/pkg/detectors/foursquare/foursquare_test.go
index 237c18a79..38cdbb842 100644
--- a/pkg/detectors/foursquare/foursquare_test.go
+++ b/pkg/detectors/foursquare/foursquare_test.go
@@ -16,7 +16,7 @@ import (
func TestFoursquare_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/frameio/frameio_test.go b/pkg/detectors/frameio/frameio_test.go
index d202eb360..86cf0a4e3 100644
--- a/pkg/detectors/frameio/frameio_test.go
+++ b/pkg/detectors/frameio/frameio_test.go
@@ -16,7 +16,7 @@ import (
func TestFrameio_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/freshbooks/freshbooks_test.go b/pkg/detectors/freshbooks/freshbooks_test.go
index 2e06425d1..8c866f0a2 100644
--- a/pkg/detectors/freshbooks/freshbooks_test.go
+++ b/pkg/detectors/freshbooks/freshbooks_test.go
@@ -16,7 +16,7 @@ import (
func TestFreshbooks_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/freshdesk/freshdesk_test.go b/pkg/detectors/freshdesk/freshdesk_test.go
index eef4ed8e3..8835cb555 100644
--- a/pkg/detectors/freshdesk/freshdesk_test.go
+++ b/pkg/detectors/freshdesk/freshdesk_test.go
@@ -16,7 +16,7 @@ import (
func TestFreshdesk_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/front/front_test.go b/pkg/detectors/front/front_test.go
index 12a2241a7..273a549b1 100644
--- a/pkg/detectors/front/front_test.go
+++ b/pkg/detectors/front/front_test.go
@@ -16,7 +16,7 @@ import (
func TestFront_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fulcrum/fulcrum_test.go b/pkg/detectors/fulcrum/fulcrum_test.go
index 4ad178e63..f75e8e7d5 100644
--- a/pkg/detectors/fulcrum/fulcrum_test.go
+++ b/pkg/detectors/fulcrum/fulcrum_test.go
@@ -16,7 +16,7 @@ import (
func TestFulcrum_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fullstory/fullstory_test.go b/pkg/detectors/fullstory/fullstory_test.go
index 83f50835b..053d4f402 100644
--- a/pkg/detectors/fullstory/fullstory_test.go
+++ b/pkg/detectors/fullstory/fullstory_test.go
@@ -16,7 +16,7 @@ import (
func TestFullstory_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fusebill/fusebill_test.go b/pkg/detectors/fusebill/fusebill_test.go
index 36029356c..f8ea9f1cd 100644
--- a/pkg/detectors/fusebill/fusebill_test.go
+++ b/pkg/detectors/fusebill/fusebill_test.go
@@ -16,7 +16,7 @@ import (
func TestFusebill_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/fxmarket/fxmarket_test.go b/pkg/detectors/fxmarket/fxmarket_test.go
index e91b2ea2d..15d1e93b3 100644
--- a/pkg/detectors/fxmarket/fxmarket_test.go
+++ b/pkg/detectors/fxmarket/fxmarket_test.go
@@ -16,7 +16,7 @@ import (
func TestFxmarket_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.go
new file mode 100644
index 000000000..fdf86b791
--- /dev/null
+++ b/pkg/detectors/gcp/gcp.go
@@ -0,0 +1,106 @@
+package gcp
+
+import (
+ "context"
+ "encoding/json"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+ "golang.org/x/oauth2/google"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(`\{[^{]+auth_provider_x509_cert_url[^}]+\}`)
+)
+
+type gcpKey struct {
+ Type string `json:"type"`
+ ProjectID string `json:"project_id"`
+ PrivateKeyID string `json:"private_key_id"`
+ PrivateKey string `json:"private_key"`
+ ClientEmail string `json:"client_email"`
+ ClientID string `json:"client_id"`
+ AuthURI string `json:"auth_uri"`
+ TokenURI string `json:"token_uri"`
+ AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`
+ ClientX509CertURL string `json:"client_x509_cert_url"`
+}
+
+func trimCarrots(s string) string {
+ s = strings.TrimPrefix(s, "<")
+ s = strings.TrimSuffix(s, ">")
+ return s
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"provider_x509"}
+}
+
+// FromData will find and optionally verify GCP secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllString(dataStr, -1)
+
+ for _, match := range matches {
+ key := match
+
+ key = strings.ReplaceAll(key, `,\\n`, `\n`)
+ key = strings.ReplaceAll(key, `\"\\n`, `\n`)
+ key = strings.ReplaceAll(key, `\\"`, `"`)
+
+ creds := gcpKey{}
+ err := json.Unmarshal([]byte(key), &creds)
+ if err != nil {
+ continue
+ }
+
+ // for Slack mangling (mailto scheme and hyperlinks)
+ if strings.Contains(creds.ClientEmail, `<mailto:`) {
+ creds.ClientEmail = strings.Split(strings.Split(creds.ClientEmail, `<mailto:`)[1], `|`)[0]
+ }
+ creds.AuthProviderX509CertURL = trimCarrots(creds.AuthProviderX509CertURL)
+ creds.AuthURI = trimCarrots(creds.AuthURI)
+ creds.ClientX509CertURL = trimCarrots(creds.ClientX509CertURL)
+ creds.TokenURI = trimCarrots(creds.TokenURI)
+
+ // Not sure why this might happen, but we've observed this with a verified cred
+ raw := []byte(creds.ClientEmail)
+ if len(raw) == 0 {
+ raw = []byte(key)
+ }
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_GCP,
+ Raw: raw,
+ Redacted: creds.ClientEmail,
+ }
+
+ if verify {
+ credBytes, _ := json.Marshal(creds)
+ credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform")
+ if err != nil {
+ continue
+ }
+ if credentials != nil {
+ _, err = credentials.TokenSource.Token()
+ if err == nil {
+ s.Verified = true
+ }
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/gcp/gcp_test.go b/pkg/detectors/gcp/gcp_test.go
new file mode 100644
index 000000000..685f3bd49
--- /dev/null
+++ b/pkg/detectors/gcp/gcp_test.go
@@ -0,0 +1,113 @@
+package gcp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGCP_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("GCP_SECRET")
+ secretInactive := testSecrets.MustGetField("GCP_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gcp secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_GCP,
+ Verified: true,
+ Redacted: "[email protected]",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, not verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gcp secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_GCP,
+ Verified: false,
+ Redacted: "secretcom",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GCP.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("GCP.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/geckoboard/geckoboard_test.go b/pkg/detectors/geckoboard/geckoboard_test.go
index 6258bb129..d96c4bfa3 100644
--- a/pkg/detectors/geckoboard/geckoboard_test.go
+++ b/pkg/detectors/geckoboard/geckoboard_test.go
@@ -16,7 +16,7 @@ import (
func TestGeckoboard_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/generic/generic.go b/pkg/detectors/generic/generic.go
new file mode 100644
index 000000000..d15d85edb
--- /dev/null
+++ b/pkg/detectors/generic/generic.go
@@ -0,0 +1,139 @@
+package generic
+
+// cat scanner/pkg/secrets/generic/top-1000.txt | awk 'length($0)>5' > scanner/pkg/secrets/generic/words.txt
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func New() Scanner {
+ excludePatterns := []string{
+ `[0-9A-Fa-f]{8}(?:-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}`, // UUID
+ `[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-4[a-fA-F0-9]{3}-[8|9|aA|bB][a-fA-F0-9]{3}-[a-fA-F0-9]{12}`, // UUIDv4
+ `[A-Z]{2,6}\-[0-9]{2,6}`, // issue tracker
+ `#[a-fA-F0-9]{6}\b`, // hex color code
+ `\b[A-Fa-f0-9]{64}\b`, // hex encoded hash
+ `\b[A-Fa-f0-9]{32}\b`, // hex encoded hash
+ `https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)`, // http
+ `\b([/]{0,1}([\w]+[/])+[\w\.]*)\b`, // filepath
+ `([0-9A-F]{2}[:-]){5}([0-9A-F]{2})`, // MAC addr
+ `\d{4}[-/]{1}([0]\d|1[0-2])[-/]{1}([0-2]\d|3[01])`, // date
+ `[v|\-]\d\.\d`, //version
+ `\d\.\d\.\d-`, //version
+ `[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}\.[\d]{1,3}`, // IPs and OIDs
+ `\b[A-Fa-f0-9x]{6,99}\b`, // hex encoding
+ `[A-Fa-f0-9x]{2}:[A-Fa-f0-9x]{2}:[A-Fa-f0-9x]{2}`, // hex encoding
+ `[\w]+\([\w, ]+\)`, // function
+ }
+
+ excludeMatchers := []*regexp.Regexp{}
+ for _, pat := range excludePatterns {
+ excludeMatchers = append(excludeMatchers, regexp.MustCompile(pat))
+ }
+
+ return Scanner{
+ excludeMatchers: excludeMatchers,
+ }
+}
+
+type Scanner struct {
+ wordList []string
+ badList []string
+ programmingBookWords []string
+ excludeMatchers []*regexp.Regexp
+}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var keywords = []string{"pass", "token", "cred", "secret", "key"}
+
+var (
+ // \x21-\x7e == ASCII 33 (0x21) and 126 (0x7e)
+ keyPat = regexp.MustCompile(detectors.PrefixRegex(keywords) + `(\b[\x21-\x7e]{16,64}\b)`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return keywords
+}
+
+// FromData will find and optionally verify Generic secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+
+ token := match[1]
+
+ // Least expensive-> most expensive filters.
+ // Substrings, then patterns.
+
+ if detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+
+ // toss any that match regexes
+ if hasReMatch(s.excludeMatchers, token) {
+ continue
+ }
+
+ // clean up containment chars
+ token = strings.Trim(token, fmt.Sprintf(`%s" '.,)(][}{`, "`"))
+
+ // toss any that b64 decode
+ // TODO: run them through again?
+ _, err := base64.StdEncoding.DecodeString(token)
+ if err == nil {
+ continue
+ }
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Generic,
+ Raw: []byte(token),
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
+
+func hasReMatch(matchers []*regexp.Regexp, token string) bool {
+ for _, m := range matchers {
+ if m.MatchString(token) {
+ return true
+ }
+ }
+ return false
+}
+
+func hasDictWord(wordList []string, token string) bool {
+ lower := strings.ToLower(token)
+ for _, word := range wordList {
+ if strings.Contains(lower, word) {
+ return true
+ }
+ }
+ return false
+}
+
+func bytesToCleanWordList(data []byte) []string {
+ words := []string{}
+ for _, word := range strings.Split(string(data), "\n") {
+ if strings.TrimSpace(word) != "" {
+ words = append(words, strings.TrimSpace(strings.ToLower(word)))
+ }
+ }
+ return words
+}
diff --git a/pkg/detectors/generic/generic_test.go b/pkg/detectors/generic/generic_test.go
new file mode 100644
index 000000000..5b27cf948
--- /dev/null
+++ b/pkg/detectors/generic/generic_test.go
@@ -0,0 +1,57 @@
+package generic
+
+import (
+ "context"
+ "testing"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+)
+
+func TestGeneric_FromChunk(t *testing.T) {
+ ctx := context.Background()
+ s := New()
+
+ found := []string{
+ "export CONFIG_SERVICE_PASSWORD=34t98hofi2309pr230",
+ "CONFIG_SERVICE_PASSWORD: 34t98hofi2309pr230",
+ "the secret is 34t98hofi2309pr230", // keyword within distance of cred-looking token
+ }
+ notFound := []string{
+ "export MAGIC_VAR=piggymetrics201925", // key does not have keyword
+ "export CONFIG_SERVICE_PASSWORD=testcredentials", // has test, cred
+ "export CONFIG_SERVICE_PASSWORD=password123", // has pass
+ "export CONFIG_SERVICE_PASSWORD=too-plain", // no digit
+ "export CONFIG_SERVICE_PASSWORD=abcdefg123", // known FP
+ "SECRET: mountain422", // excluded by word list
+ "secret_guid=3fc0b7f7-da09-4ae7-a9c8-d69824b1819b", // excluded by matcher
+ "secret_issue_key: BLAH-23490", // excluded by matcher
+ }
+
+ for _, data := range found {
+ got, _ := s.FromData(ctx, false, []byte(data))
+ if len(got) == 0 {
+ t.Errorf("Generic.FromData() expected secret for data: %s", data)
+ return
+ }
+ }
+
+ for _, data := range notFound {
+ got, _ := s.FromData(ctx, false, []byte(data))
+ if len(got) != 0 {
+ t.Errorf("Generic.FromData() expected no secret for data: %s", data)
+ return
+ }
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := New()
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/gengo/gengo.go b/pkg/detectors/gengo/gengo.go
new file mode 100644
index 000000000..89c3b22c2
--- /dev/null
+++ b/pkg/detectors/gengo/gengo.go
@@ -0,0 +1,118 @@
+package gengo
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha1"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "strconv"
+ "time"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ //Removed bounds since there are some cases where the start and end of the token is a special character
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)
+ secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"gengo"}
+}
+
+// FromData will find and optionally verify Gengo secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ for _, secretMatch := range secretMatches {
+ if len(secretMatch) != 2 {
+ continue
+ }
+ resSecretMatch := strings.TrimSpace(secretMatch[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Gengo,
+ Raw: []byte(resSecretMatch),
+ }
+
+ if verify {
+
+ timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+ signature := getGengoSignature(timestamp, resSecretMatch)
+
+ req, _ := http.NewRequest("GET", fmt.Sprintf("http://api.gengo.com/v2/account/me?ts=%s&api_key=%s&api_sig=%s", timestamp, resMatch, signature), nil)
+ req.Header.Add("Accept", "application/json")
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ body, errBody := ioutil.ReadAll(res.Body)
+
+ if errBody == nil {
+ var response Response
+ json.Unmarshal(body, &response)
+
+ if res.StatusCode >= 200 && res.StatusCode < 300 && response.OpStat == "ok" {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+
+ if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+ }
+
+ return detectors.CleanResults(results), nil
+}
+
+type Response struct {
+ OpStat string `json:"opstat"`
+}
+
+func getGengoSignature(timeStamp string, secret string) string {
+
+ mac := hmac.New(sha1.New, []byte(secret))
+ mac.Write([]byte(timeStamp))
+ macsum := mac.Sum(nil)
+ return hex.EncodeToString(macsum)
+}
diff --git a/pkg/detectors/gengo/gengo_test.go b/pkg/detectors/gengo/gengo_test.go
new file mode 100644
index 000000000..4fff0ebb5
--- /dev/null
+++ b/pkg/detectors/gengo/gengo_test.go
@@ -0,0 +1,115 @@
+package gengo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGengo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ key := testSecrets.MustGetField("GENGO_KEY")
+ inactiveKey := testSecrets.MustGetField("GENGO_KEY_INACTIVE")
+ secret := testSecrets.MustGetField("GENGO")
+ inactiveSecret := testSecrets.MustGetField("GENGO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gengo key %s with gengo secret %s within", key, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Gengo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gengo key %s with gengo secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Gengo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Gengo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Gengo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/geoapify/geoapify_test.go b/pkg/detectors/geoapify/geoapify_test.go
index 61cad9426..c64d5358d 100644
--- a/pkg/detectors/geoapify/geoapify_test.go
+++ b/pkg/detectors/geoapify/geoapify_test.go
@@ -16,7 +16,7 @@ import (
func TestGeoapify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/geocode/geocode_test.go b/pkg/detectors/geocode/geocode_test.go
index 904998d18..698c58730 100644
--- a/pkg/detectors/geocode/geocode_test.go
+++ b/pkg/detectors/geocode/geocode_test.go
@@ -16,7 +16,7 @@ import (
func TestGeocode_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/geocodify/geocodify_test.go b/pkg/detectors/geocodify/geocodify_test.go
index 943779374..e38ed2768 100644
--- a/pkg/detectors/geocodify/geocodify_test.go
+++ b/pkg/detectors/geocodify/geocodify_test.go
@@ -16,7 +16,7 @@ import (
func TestGeocodify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/geocodio/geocodio_test.go b/pkg/detectors/geocodio/geocodio_test.go
index 38eb036f3..8adca4be9 100644
--- a/pkg/detectors/geocodio/geocodio_test.go
+++ b/pkg/detectors/geocodio/geocodio_test.go
@@ -16,7 +16,7 @@ import (
func TestGeocodio_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/geoipifi/geoipifi_test.go b/pkg/detectors/geoipifi/geoipifi_test.go
index 586a1c88b..1689ae691 100644
--- a/pkg/detectors/geoipifi/geoipifi_test.go
+++ b/pkg/detectors/geoipifi/geoipifi_test.go
@@ -16,7 +16,7 @@ import (
func TestGeoIpifi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/getemail/getemail_test.go b/pkg/detectors/getemail/getemail_test.go
index 4598d5e68..6c21905f3 100644
--- a/pkg/detectors/getemail/getemail_test.go
+++ b/pkg/detectors/getemail/getemail_test.go
@@ -16,7 +16,7 @@ import (
func TestGetEmail_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/getemails/getemails_test.go b/pkg/detectors/getemails/getemails_test.go
index ac4c30547..34853c49e 100644
--- a/pkg/detectors/getemails/getemails_test.go
+++ b/pkg/detectors/getemails/getemails_test.go
@@ -16,7 +16,7 @@ import (
func TestGetEmails_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/getgeoapi/getgeoapi_test.go b/pkg/detectors/getgeoapi/getgeoapi_test.go
index c70e3b835..66aeafc63 100644
--- a/pkg/detectors/getgeoapi/getgeoapi_test.go
+++ b/pkg/detectors/getgeoapi/getgeoapi_test.go
@@ -16,7 +16,7 @@ import (
func TestGetgeoapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/getgist/getgist_test.go b/pkg/detectors/getgist/getgist_test.go
index 32b15455f..31ed98900 100644
--- a/pkg/detectors/getgist/getgist_test.go
+++ b/pkg/detectors/getgist/getgist_test.go
@@ -16,7 +16,7 @@ import (
func TestGetgist_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/getsandbox/getsandbox_test.go b/pkg/detectors/getsandbox/getsandbox_test.go
index 147b348ac..034caa30b 100644
--- a/pkg/detectors/getsandbox/getsandbox_test.go
+++ b/pkg/detectors/getsandbox/getsandbox_test.go
@@ -16,7 +16,7 @@ import (
func TestGetSandbox_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.go
new file mode 100644
index 000000000..45ebf9f9b
--- /dev/null
+++ b/pkg/detectors/github/github.go
@@ -0,0 +1,96 @@
+package github
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ // Oauth token
+ // https://developer.github.com/v3/#oauth2-token-sent-in-a-header
+ // Token type list:
+ // https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
+ keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,255}\b)`)
+
+ //TODO: Oauth2 client_id and client_secret
+ // https://developer.github.com/v3/#oauth2-keysecret
+)
+
+// TODO: Add secret context?? Information about access, ownership etc
+type userRes struct {
+ Login string `json:"login"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ Name string `json:"name"`
+ Company string `json:"company"`
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"ghp_", "gho_", "ghu_", "ghs_", "ghr_"}
+}
+
+// FromData will find and optionally verify GitHub secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ // First match is entire regex, second is the first group.
+ if len(match) != 2 {
+ continue
+ }
+
+ token := match[1]
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Github,
+ Raw: []byte(token),
+ }
+
+ if verify {
+ client := common.RetryableHttpClient()
+ // https://developer.github.com/v3/users/#get-the-authenticated-user
+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
+ res, err := client.Do(req)
+ if err != nil {
+ break
+ }
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ var userResponse userRes
+ err = json.NewDecoder(res.Body).Decode(&userResponse)
+ if err == nil {
+ s.Verified = true
+ }
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/github/github_test.go b/pkg/detectors/github/github_test.go
new file mode 100644
index 000000000..c696ac417
--- /dev/null
+++ b/pkg/detectors/github/github_test.go
@@ -0,0 +1,180 @@
+package github
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGitHub_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ unverifiedGhp := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHP")
+ unverifiedGhpLong := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHP_LONG")
+ unverifiedGho := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHO")
+ unverifiedGhu := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHU")
+ unverifiedGhs := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHS")
+ unverifiedGhr := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHR")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, unverified ghp",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhp)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified gho",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGho)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified ghu",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhu)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified ghs",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhs)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified ghr",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", unverifiedGhr)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified ghp future length 255",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(unverifiedGhpLong),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("https://raw.github.com/k/d890e8640f20fba3215ba7be8e0ff145aeb8c17c/include/base64.js"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GitHub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("GitHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.go
new file mode 100644
index 000000000..5abdf41ce
--- /dev/null
+++ b/pkg/detectors/github_old/github_old.go
@@ -0,0 +1,92 @@
+package github_old
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ // Oauth token
+ // https://developer.github.com/v3/#oauth2-token-sent-in-a-header
+ keyPat = regexp.MustCompile(`(?i)(?:github)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
+
+ //TODO: Oauth2 client_id and client_secret
+ // https://developer.github.com/v3/#oauth2-keysecret
+)
+
+// TODO: Add secret context?? Information about access, ownership etc
+type userRes struct {
+ Login string `json:"login"`
+ Type string `json:"type"`
+ SiteAdmin bool `json:"site_admin"`
+ Name string `json:"name"`
+ Company string `json:"company"`
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"github"}
+}
+
+// FromData will find and optionally verify GitHub secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ // First match is entire regex, second is the first group.
+ if len(match) != 2 {
+ continue
+ }
+
+ token := match[1]
+
+ specificFPs := []detectors.FalsePositive{"github commit"}
+ if detectors.IsKnownFalsePositive(token, specificFPs, false) {
+ continue
+ }
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Github,
+ Raw: []byte(token),
+ }
+
+ if verify {
+ client := common.RetryableHttpClient()
+ // https://developer.github.com/v3/users/#get-the-authenticated-user
+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
+ res, err := client.Do(req)
+ if err != nil {
+ break
+ }
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ var userResponse userRes
+ err = json.NewDecoder(res.Body).Decode(&userResponse)
+ if err == nil {
+ s.Verified = true
+ }
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/github_old/github_old_test.go b/pkg/detectors/github_old/github_old_test.go
new file mode 100644
index 000000000..97e5a5c82
--- /dev/null
+++ b/pkg/detectors/github_old/github_old_test.go
@@ -0,0 +1,122 @@
+package github_old
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGitHub_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("GITHUB_OLD")
+ secretInactive := testSecrets.MustGetField("GITHUB_OLD_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a github secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Github,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found url",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("https://raw.github.com/k/d890e8640f20fba3215ba7be8e0ff145aeb8c17c/include/base64.js"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "not found ref",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("https://github.com/lz4/lz4 @ dccf8826f1d76efcbdc655e63cc04cdbd1123619"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GitHub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("GitHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/githubapp/githubapp_test.go b/pkg/detectors/githubapp/githubapp_test.go
index dad95c35f..8b3bc7e60 100644
--- a/pkg/detectors/githubapp/githubapp_test.go
+++ b/pkg/detectors/githubapp/githubapp_test.go
@@ -17,7 +17,7 @@ import (
func TestGitHubApp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.go
new file mode 100644
index 000000000..9e232eb38
--- /dev/null
+++ b/pkg/detectors/gitlab/gitlab.go
@@ -0,0 +1,82 @@
+package gitlab
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b([a-zA-Z0-9\-=_]{20,22})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"gitlab"}
+}
+
+// FromData will find and optionally verify Gitlab secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+
+ secret := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Gitlab,
+ Raw: []byte(match[1]),
+ }
+
+ if verify {
+ // there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
+ // they all grant access to different parts of the API. I couldn't find an endpoint that every
+ // one of these scopes has access to, so we just check an example endpoint for each scope. If any
+ // of them contain data, we know we have a valid key, but if they all fail, we don't
+ baseURL := "https://gitlab.com/api/v4"
+
+ client := common.RetryableHttpClient()
+
+ // test `read_user` scope
+ req, _ := http.NewRequest("GET", baseURL+"/user", nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ // 200 means good key and has `read_user` scope
+ // 403 means good key but not the right scope
+ // 401 is bad key
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ secret.Verified = true
+ }
+ }
+
+ if !secret.Verified {
+ if detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, secret)
+ }
+
+ return
+}
diff --git a/pkg/detectors/gitlab/gitlab_test.go b/pkg/detectors/gitlab/gitlab_test.go
new file mode 100644
index 000000000..f37eeb455
--- /dev/null
+++ b/pkg/detectors/gitlab/gitlab_test.go
@@ -0,0 +1,111 @@
+package gitlab
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGitlab_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("GITLAB")
+ secretInactive := testSecrets.MustGetField("GITLAB_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Gitlab,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gitlab secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Gitlab,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.go
new file mode 100644
index 000000000..265c4bf73
--- /dev/null
+++ b/pkg/detectors/gitlabv2/gitlab.go
@@ -0,0 +1,82 @@
+package gitlabv2
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"glpat-"}
+}
+
+// FromData will find and optionally verify Gitlab secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+
+ secret := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Gitlab,
+ Raw: []byte(match[1]),
+ }
+
+ if verify {
+ // there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
+ // they all grant access to different parts of the API. I couldn't find an endpoint that every
+ // one of these scopes has access to, so we just check an example endpoint for each scope. If any
+ // of them contain data, we know we have a valid key, but if they all fail, we don't
+ baseURL := "https://gitlab.com/api/v4"
+
+ client := common.RetryableHttpClient()
+
+ // test `read_user` scope
+ req, _ := http.NewRequest("GET", baseURL+"/user", nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ // 200 means good key and has `read_user` scope
+ // 403 means good key but not the right scope
+ // 401 is bad key
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ secret.Verified = true
+ }
+ }
+
+ if !secret.Verified {
+ if detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, secret)
+ }
+
+ return
+}
diff --git a/pkg/detectors/gitlabv2/gitlab_test.go b/pkg/detectors/gitlabv2/gitlab_test.go
new file mode 100644
index 000000000..3782b5cdf
--- /dev/null
+++ b/pkg/detectors/gitlabv2/gitlab_test.go
@@ -0,0 +1,107 @@
+package gitlabv2
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGitlab_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("GITLABV2")
+ secretInactive := testSecrets.MustGetField("GITLABV2_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Gitlab,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/gitter/gitter_test.go b/pkg/detectors/gitter/gitter_test.go
index a93e703cd..a11ca3bb4 100644
--- a/pkg/detectors/gitter/gitter_test.go
+++ b/pkg/detectors/gitter/gitter_test.go
@@ -16,7 +16,7 @@ import (
func TestGitter_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/glassnode/glassnode_test.go b/pkg/detectors/glassnode/glassnode_test.go
index d92d7043c..e4e1f2075 100644
--- a/pkg/detectors/glassnode/glassnode_test.go
+++ b/pkg/detectors/glassnode/glassnode_test.go
@@ -16,7 +16,7 @@ import (
func TestGlassnode_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/gocardless/gocardless.go b/pkg/detectors/gocardless/gocardless.go
new file mode 100644
index 000000000..a6da73df8
--- /dev/null
+++ b/pkg/detectors/gocardless/gocardless.go
@@ -0,0 +1,77 @@
+package gocardless
+
+import (
+ "context"
+ "fmt"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ //Removed bounds at the end of the regex since there are some cases that the token ends with an underscore (_) or a dash (-)
+ keyPat = regexp.MustCompile(`\b(live_[0-9A-Za-z\_\-]{40}[ "'\r\n]{1})`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"gocardless"}
+}
+
+// FromData will find and optionally verify GoCardless secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_GoCardless,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://api.gocardless.com/customers/", nil)
+ req.Header.Add("Content-Type", "application/json")
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+ req.Header.Add("GoCardless-Version", "2015-07-06")
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/gocardless/gocardless_test.go b/pkg/detectors/gocardless/gocardless_test.go
new file mode 100644
index 000000000..c384552d9
--- /dev/null
+++ b/pkg/detectors/gocardless/gocardless_test.go
@@ -0,0 +1,113 @@
+package gocardless
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGoCardless_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("GOCARDLESS")
+ inactiveSecret := testSecrets.MustGetField("GOCARDLESS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gocardless secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_GoCardless,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a gocardless secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_GoCardless,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GoCardless.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("GoCardless.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/goodday/goodday.go b/pkg/detectors/goodday/goodday.go
new file mode 100644
index 000000000..fa081eaa3
--- /dev/null
+++ b/pkg/detectors/goodday/goodday.go
@@ -0,0 +1,73 @@
+package goodday
+
+import (
+ "context"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"goodday"}) + `\b([a-z0-9]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"goodday"}
+}
+
+// FromData will find and optionally verify GoodDay secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_GoodDay,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://api.goodday.work/2.0/users", nil)
+ req.Header.Add("gd-api-token", resMatch)
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/goodday/goodday_test.go b/pkg/detectors/goodday/goodday_test.go
new file mode 100644
index 000000000..3fb90bd48
--- /dev/null
+++ b/pkg/detectors/goodday/goodday_test.go
@@ -0,0 +1,113 @@
+package goodday
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestGoodDay_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("GOODDAY")
+ inactiveSecret := testSecrets.MustGetField("GOODDAY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a goodday secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_GoodDay,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a goodday secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_GoodDay,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("GoodDay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("GoodDay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/graphcms/graphcms_test.go b/pkg/detectors/graphcms/graphcms_test.go
index bc29c051f..7a0380ba3 100644
--- a/pkg/detectors/graphcms/graphcms_test.go
+++ b/pkg/detectors/graphcms/graphcms_test.go
@@ -16,7 +16,7 @@ import (
func TestGraphCMS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/graphhopper/graphhopper_test.go b/pkg/detectors/graphhopper/graphhopper_test.go
index 8baa72743..f79aaa922 100644
--- a/pkg/detectors/graphhopper/graphhopper_test.go
+++ b/pkg/detectors/graphhopper/graphhopper_test.go
@@ -16,7 +16,7 @@ import (
func TestGraphhopper_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/groovehq/groovehq_test.go b/pkg/detectors/groovehq/groovehq_test.go
index be1d7ebff..befb0c5e6 100644
--- a/pkg/detectors/groovehq/groovehq_test.go
+++ b/pkg/detectors/groovehq/groovehq_test.go
@@ -16,7 +16,7 @@ import (
func TestGroovehq_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/guardianapi/guardianapi_test.go b/pkg/detectors/guardianapi/guardianapi_test.go
index fed330d15..cb4f53a91 100644
--- a/pkg/detectors/guardianapi/guardianapi_test.go
+++ b/pkg/detectors/guardianapi/guardianapi_test.go
@@ -16,7 +16,7 @@ import (
func TestGuardianapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/guru/guru_test.go b/pkg/detectors/guru/guru_test.go
index 3fedf3bb0..54bad8df7 100644
--- a/pkg/detectors/guru/guru_test.go
+++ b/pkg/detectors/guru/guru_test.go
@@ -16,7 +16,7 @@ import (
func TestGuru_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/gyazo/gyazo_test.go b/pkg/detectors/gyazo/gyazo_test.go
index 1ed8fe1bc..621d330e5 100644
--- a/pkg/detectors/gyazo/gyazo_test.go
+++ b/pkg/detectors/gyazo/gyazo_test.go
@@ -16,7 +16,7 @@ import (
func TestGyazo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/happi/happi_test.go b/pkg/detectors/happi/happi_test.go
index 287b306b2..39f035404 100644
--- a/pkg/detectors/happi/happi_test.go
+++ b/pkg/detectors/happi/happi_test.go
@@ -16,7 +16,7 @@ import (
func TestHappi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/happyscribe/happyscribe_test.go b/pkg/detectors/happyscribe/happyscribe_test.go
index 6b19f0c1d..ec9a35c43 100644
--- a/pkg/detectors/happyscribe/happyscribe_test.go
+++ b/pkg/detectors/happyscribe/happyscribe_test.go
@@ -16,7 +16,7 @@ import (
func TestHappyscribe_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/harvest/harvest_test.go b/pkg/detectors/harvest/harvest_test.go
index 06bcc84d3..f4392b224 100644
--- a/pkg/detectors/harvest/harvest_test.go
+++ b/pkg/detectors/harvest/harvest_test.go
@@ -16,7 +16,7 @@ import (
func TestHarvest_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/hellosign/hellosign_test.go b/pkg/detectors/hellosign/hellosign_test.go
index f45246310..92f860c87 100644
--- a/pkg/detectors/hellosign/hellosign_test.go
+++ b/pkg/detectors/hellosign/hellosign_test.go
@@ -16,7 +16,7 @@ import (
func TestHelloSign_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/helpcrunch/helpcrunch_test.go b/pkg/detectors/helpcrunch/helpcrunch_test.go
index 806657d20..33c542c62 100644
--- a/pkg/detectors/helpcrunch/helpcrunch_test.go
+++ b/pkg/detectors/helpcrunch/helpcrunch_test.go
@@ -16,7 +16,7 @@ import (
func TestHelpCrunch_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/helpscout/helpscout_test.go b/pkg/detectors/helpscout/helpscout_test.go
index c9ac1e285..0ffef8fb1 100644
--- a/pkg/detectors/helpscout/helpscout_test.go
+++ b/pkg/detectors/helpscout/helpscout_test.go
@@ -16,7 +16,7 @@ import (
func TestHelpscout_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/hereapi/hereapi_test.go b/pkg/detectors/hereapi/hereapi_test.go
index c9b71c6ca..1567ea386 100644
--- a/pkg/detectors/hereapi/hereapi_test.go
+++ b/pkg/detectors/hereapi/hereapi_test.go
@@ -16,7 +16,7 @@ import (
func TestHereAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/heroku/heroku_test.go b/pkg/detectors/heroku/heroku_test.go
index 2ab4139a4..ce8604831 100644
--- a/pkg/detectors/heroku/heroku_test.go
+++ b/pkg/detectors/heroku/heroku_test.go
@@ -16,7 +16,7 @@ import (
func TestHeroku_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/hive/hive_test.go b/pkg/detectors/hive/hive_test.go
index 46f642615..f146fff1c 100644
--- a/pkg/detectors/hive/hive_test.go
+++ b/pkg/detectors/hive/hive_test.go
@@ -16,7 +16,7 @@ import (
func TestHive_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/hiveage/hiveage_test.go b/pkg/detectors/hiveage/hiveage_test.go
index b257d8b40..bcefbe78a 100644
--- a/pkg/detectors/hiveage/hiveage_test.go
+++ b/pkg/detectors/hiveage/hiveage_test.go
@@ -16,7 +16,7 @@ import (
func TestHiveage_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/holidayapi/holidayapi_test.go b/pkg/detectors/holidayapi/holidayapi_test.go
index 9510a0d65..55cb3afc0 100644
--- a/pkg/detectors/holidayapi/holidayapi_test.go
+++ b/pkg/detectors/holidayapi/holidayapi_test.go
@@ -16,7 +16,7 @@ import (
func TestHolidayAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/host/host_test.go b/pkg/detectors/host/host_test.go
index ff45f3e08..ac62e88ab 100644
--- a/pkg/detectors/host/host_test.go
+++ b/pkg/detectors/host/host_test.go
@@ -16,7 +16,7 @@ import (
func TestHost_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/html2pdf/html2pdf_test.go b/pkg/detectors/html2pdf/html2pdf_test.go
index 2ad8b1523..4a08a779c 100644
--- a/pkg/detectors/html2pdf/html2pdf_test.go
+++ b/pkg/detectors/html2pdf/html2pdf_test.go
@@ -16,7 +16,7 @@ import (
func TestHtml2Pdf_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/hubspotapikey/hubspotapikey_test.go b/pkg/detectors/hubspotapikey/hubspotapikey_test.go
index f5c4cf6e5..1a4261be4 100644
--- a/pkg/detectors/hubspotapikey/hubspotapikey_test.go
+++ b/pkg/detectors/hubspotapikey/hubspotapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestHubSpotApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/humanity/humanity_test.go b/pkg/detectors/humanity/humanity_test.go
index 3ae82e41c..1ba17ad6f 100644
--- a/pkg/detectors/humanity/humanity_test.go
+++ b/pkg/detectors/humanity/humanity_test.go
@@ -16,7 +16,7 @@ import (
func TestHumanity_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/hunter/hunter_test.go b/pkg/detectors/hunter/hunter_test.go
index 1925a6899..1eecfd4fa 100644
--- a/pkg/detectors/hunter/hunter_test.go
+++ b/pkg/detectors/hunter/hunter_test.go
@@ -16,7 +16,7 @@ import (
func TestHunter_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
@@ -56,7 +56,7 @@ func TestHunter_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a hunter secret %s within", secretInactive)),
+ data: []byte(fmt.Sprintf("You can find a hunter secret %s within but not valid", secretInactive)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
diff --git a/pkg/detectors/hypertrack/hypertrack_test.go b/pkg/detectors/hypertrack/hypertrack_test.go
index b4e83e94f..5fc157f86 100644
--- a/pkg/detectors/hypertrack/hypertrack_test.go
+++ b/pkg/detectors/hypertrack/hypertrack_test.go
@@ -16,7 +16,7 @@ import (
func TestHypertrack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go b/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go
index 23b80d9d5..3860719ba 100644
--- a/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go
+++ b/pkg/detectors/ibmclouduserkey/ibmclouduserkey_test.go
@@ -16,7 +16,7 @@ import (
func TestIbmCloudUserKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/iconfinder/iconfinder_test.go b/pkg/detectors/iconfinder/iconfinder_test.go
index 5ae4486c6..cf05df130 100644
--- a/pkg/detectors/iconfinder/iconfinder_test.go
+++ b/pkg/detectors/iconfinder/iconfinder_test.go
@@ -16,7 +16,7 @@ import (
func TestIconFinder_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/iexcloud/iexcloud_test.go b/pkg/detectors/iexcloud/iexcloud_test.go
index ad191ca57..2846dee42 100644
--- a/pkg/detectors/iexcloud/iexcloud_test.go
+++ b/pkg/detectors/iexcloud/iexcloud_test.go
@@ -16,7 +16,7 @@ import (
func TestIexcloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/imagekit/imagekit_test.go b/pkg/detectors/imagekit/imagekit_test.go
index 60b51d735..869c6f6f8 100644
--- a/pkg/detectors/imagekit/imagekit_test.go
+++ b/pkg/detectors/imagekit/imagekit_test.go
@@ -16,7 +16,7 @@ import (
func TestImagekit_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/imagga/imagga_test.go b/pkg/detectors/imagga/imagga_test.go
index 11d6c9539..ae938fdb2 100644
--- a/pkg/detectors/imagga/imagga_test.go
+++ b/pkg/detectors/imagga/imagga_test.go
@@ -16,7 +16,7 @@ import (
func TestImagga_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/impala/impala_test.go b/pkg/detectors/impala/impala_test.go
index 6bba820aa..4f76e1b0e 100644
--- a/pkg/detectors/impala/impala_test.go
+++ b/pkg/detectors/impala/impala_test.go
@@ -16,7 +16,7 @@ import (
func TestImpala_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/insightly/insightly_test.go b/pkg/detectors/insightly/insightly_test.go
index eff524871..361af36e3 100644
--- a/pkg/detectors/insightly/insightly_test.go
+++ b/pkg/detectors/insightly/insightly_test.go
@@ -16,7 +16,7 @@ import (
func TestInsightly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/integromat/integromat_test.go b/pkg/detectors/integromat/integromat_test.go
index abc48cc29..40b30e30d 100644
--- a/pkg/detectors/integromat/integromat_test.go
+++ b/pkg/detectors/integromat/integromat_test.go
@@ -16,7 +16,7 @@ import (
func TestIntegromat_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/intercom/intercom_test.go b/pkg/detectors/intercom/intercom_test.go
index 39e283974..bb1e65e8d 100644
--- a/pkg/detectors/intercom/intercom_test.go
+++ b/pkg/detectors/intercom/intercom_test.go
@@ -16,7 +16,7 @@ import (
func TestInterCom_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/intrinio/intrinio_test.go b/pkg/detectors/intrinio/intrinio_test.go
index 844213a49..11f370a35 100644
--- a/pkg/detectors/intrinio/intrinio_test.go
+++ b/pkg/detectors/intrinio/intrinio_test.go
@@ -16,7 +16,7 @@ import (
func TestIntrinio_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/invoiceocean/invoiceocean_test.go b/pkg/detectors/invoiceocean/invoiceocean_test.go
index 779b6ceb1..262145663 100644
--- a/pkg/detectors/invoiceocean/invoiceocean_test.go
+++ b/pkg/detectors/invoiceocean/invoiceocean_test.go
@@ -16,7 +16,7 @@ import (
func TestInvoiceocean_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ipapi/ipapi_test.go b/pkg/detectors/ipapi/ipapi_test.go
index e4f8dd8e2..9aa5bd57b 100644
--- a/pkg/detectors/ipapi/ipapi_test.go
+++ b/pkg/detectors/ipapi/ipapi_test.go
@@ -16,7 +16,7 @@ import (
func TestIpapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ipgeolocation/ipgeolocation_test.go b/pkg/detectors/ipgeolocation/ipgeolocation_test.go
index 58a21c72f..a22d18c85 100644
--- a/pkg/detectors/ipgeolocation/ipgeolocation_test.go
+++ b/pkg/detectors/ipgeolocation/ipgeolocation_test.go
@@ -16,7 +16,7 @@ import (
func TestIPGeolocation_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ipify/ipify_test.go b/pkg/detectors/ipify/ipify_test.go
index 4185afe75..87aa61af7 100644
--- a/pkg/detectors/ipify/ipify_test.go
+++ b/pkg/detectors/ipify/ipify_test.go
@@ -16,7 +16,7 @@ import (
func TestIpify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ipinfodb/ipinfodb_test.go b/pkg/detectors/ipinfodb/ipinfodb_test.go
index 6a4fd2195..0cfed7661 100644
--- a/pkg/detectors/ipinfodb/ipinfodb_test.go
+++ b/pkg/detectors/ipinfodb/ipinfodb_test.go
@@ -16,7 +16,7 @@ import (
func TestIPinfoDB_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ipquality/ipquality_test.go b/pkg/detectors/ipquality/ipquality_test.go
index a58911a7b..90a62cb47 100644
--- a/pkg/detectors/ipquality/ipquality_test.go
+++ b/pkg/detectors/ipquality/ipquality_test.go
@@ -16,7 +16,7 @@ import (
func TestIpquality_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ipstack/ipstack_test.go b/pkg/detectors/ipstack/ipstack_test.go
index 2fe814816..5fc94b5f2 100644
--- a/pkg/detectors/ipstack/ipstack_test.go
+++ b/pkg/detectors/ipstack/ipstack_test.go
@@ -16,7 +16,7 @@ import (
func TestIpStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go
new file mode 100644
index 000000000..d9be8e8d1
--- /dev/null
+++ b/pkg/detectors/jdbc/jdbc.go
@@ -0,0 +1,74 @@
+package jdbc
+
+import (
+ "context"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:\/\/\w[\s\S]{0,512}?password[=: \"']+(?P<pass>[^<{($]*?)[ \s'\"]+`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"jdbc"}
+}
+
+// FromData will find and optionally verify Jdbc secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ for _, match := range matches {
+ if match[1] == "" {
+ continue
+ }
+ token := match[0]
+ password := match[1]
+
+ //TODO if username and password are the same, username will also be redacted... I think this is probably correct.
+ redact := strings.TrimSpace(strings.Replace(token, password, strings.Repeat("*", len(password)), -1))
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Raw: []byte(token),
+ Redacted: redact,
+ }
+
+ if verify {
+ //TODO can this be verified? Possibly. Could triage verification to other DBMS strings
+ s.Verified = false
+ // client := common.RetryableHttpClient()
+ // req, _ := http.NewRequestWithContext(ctx, "GET", "https://jdbcci.com/api/v2/me", nil)
+ // req.Header.Add("Accept", "application/json;")
+ // req.Header.Add("Jdbc-Token", token)
+ // res, err := client.Do(req)
+ // if err != nil {
+ // break
+ // }
+ // if res.StatusCode >= 200 && res.StatusCode < 300 {
+ // s.Verified = true
+ // }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go
new file mode 100644
index 000000000..1dc8baec8
--- /dev/null
+++ b/pkg/detectors/jdbc/jdbc_test.go
@@ -0,0 +1,89 @@
+package jdbc
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestJdbc_FromChunk(t *testing.T) {
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(`jdbc connection string: jdbc:mysql://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword <-`),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Verified: false,
+ Redacted: "jdbc:mysql://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=************",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Jdbc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.go
index 01c96c4e6..c16f1228d 100644
--- a/pkg/detectors/jiratoken/jiratoken_test.go
+++ b/pkg/detectors/jiratoken/jiratoken_test.go
@@ -16,7 +16,7 @@ import (
func TestJiraToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/jotform/jotform_test.go b/pkg/detectors/jotform/jotform_test.go
index 6a93249d2..8793a391c 100644
--- a/pkg/detectors/jotform/jotform_test.go
+++ b/pkg/detectors/jotform/jotform_test.go
@@ -16,7 +16,7 @@ import (
func TestJotform_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/jumpcloud/jumpcloud_test.go b/pkg/detectors/jumpcloud/jumpcloud_test.go
index f1db1498b..28958e6e4 100644
--- a/pkg/detectors/jumpcloud/jumpcloud_test.go
+++ b/pkg/detectors/jumpcloud/jumpcloud_test.go
@@ -16,7 +16,7 @@ import (
func TestJumpcloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/juro/juro_test.go b/pkg/detectors/juro/juro_test.go
index fcc269b6f..4a831cc57 100644
--- a/pkg/detectors/juro/juro_test.go
+++ b/pkg/detectors/juro/juro_test.go
@@ -16,7 +16,7 @@ import (
func TestJuro_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/kanban/kanban_test.go b/pkg/detectors/kanban/kanban_test.go
index e8e2f6e26..f2ddecefc 100644
--- a/pkg/detectors/kanban/kanban_test.go
+++ b/pkg/detectors/kanban/kanban_test.go
@@ -16,7 +16,7 @@ import (
func TestKanban_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/karmacrm/karmacrm_test.go b/pkg/detectors/karmacrm/karmacrm_test.go
index c638a340e..a70f3dbc4 100644
--- a/pkg/detectors/karmacrm/karmacrm_test.go
+++ b/pkg/detectors/karmacrm/karmacrm_test.go
@@ -16,7 +16,7 @@ import (
func TestKarmaCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/keenio/keenio_test.go b/pkg/detectors/keenio/keenio_test.go
index 8f5c68d36..82e0da4d7 100644
--- a/pkg/detectors/keenio/keenio_test.go
+++ b/pkg/detectors/keenio/keenio_test.go
@@ -16,7 +16,7 @@ import (
func TestKeenIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/kickbox/kickbox_test.go b/pkg/detectors/kickbox/kickbox_test.go
index f70f27d97..840e7fc9f 100644
--- a/pkg/detectors/kickbox/kickbox_test.go
+++ b/pkg/detectors/kickbox/kickbox_test.go
@@ -16,7 +16,7 @@ import (
func TestKickbox_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/kontent/kontent_test.go b/pkg/detectors/kontent/kontent_test.go
index a6106376d..61ad0f498 100644
--- a/pkg/detectors/kontent/kontent_test.go
+++ b/pkg/detectors/kontent/kontent_test.go
@@ -16,7 +16,7 @@ import (
func TestKontent_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/kraken/kraken_test.go b/pkg/detectors/kraken/kraken_test.go
index a2df32932..24c96f4d5 100644
--- a/pkg/detectors/kraken/kraken_test.go
+++ b/pkg/detectors/kraken/kraken_test.go
@@ -16,7 +16,7 @@ import (
func TestKraken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/kylas/kylas_test.go b/pkg/detectors/kylas/kylas_test.go
index 759353654..c532f8243 100644
--- a/pkg/detectors/kylas/kylas_test.go
+++ b/pkg/detectors/kylas/kylas_test.go
@@ -16,7 +16,7 @@ import (
func TestKylas_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/languagelayer/languagelayer_test.go b/pkg/detectors/languagelayer/languagelayer_test.go
index c94c412ea..2151f0ad0 100644
--- a/pkg/detectors/languagelayer/languagelayer_test.go
+++ b/pkg/detectors/languagelayer/languagelayer_test.go
@@ -16,7 +16,7 @@ import (
func TestLanguageLayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/lastfm/lastfm_test.go b/pkg/detectors/lastfm/lastfm_test.go
index 79969d117..1117336a9 100644
--- a/pkg/detectors/lastfm/lastfm_test.go
+++ b/pkg/detectors/lastfm/lastfm_test.go
@@ -16,7 +16,7 @@ import (
func TestLastfm_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/launchdarkly/launchdarkly_test.go b/pkg/detectors/launchdarkly/launchdarkly_test.go
index a49d143e6..5189c6ab3 100644
--- a/pkg/detectors/launchdarkly/launchdarkly_test.go
+++ b/pkg/detectors/launchdarkly/launchdarkly_test.go
@@ -16,7 +16,7 @@ import (
func TestLaunchDarkly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/leadfeeder/leadfeeder_test.go b/pkg/detectors/leadfeeder/leadfeeder_test.go
index a4b877b32..7f96c6a7c 100644
--- a/pkg/detectors/leadfeeder/leadfeeder_test.go
+++ b/pkg/detectors/leadfeeder/leadfeeder_test.go
@@ -16,7 +16,7 @@ import (
func TestLeadfeeder_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go b/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go
index 766d6ef7d..b988cdaa0 100644
--- a/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go
+++ b/pkg/detectors/lessannoyingcrm/lessannoyingcrm_test.go
@@ -16,7 +16,7 @@ import (
func TestLessAnnoyingCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/lexigram/lexigram_test.go b/pkg/detectors/lexigram/lexigram_test.go
index b9a649e63..a19a097d3 100644
--- a/pkg/detectors/lexigram/lexigram_test.go
+++ b/pkg/detectors/lexigram/lexigram_test.go
@@ -16,7 +16,7 @@ import (
func TestLexigram_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/linearapi/linearapi_test.go b/pkg/detectors/linearapi/linearapi_test.go
index 3980bd2ea..eebbb517c 100644
--- a/pkg/detectors/linearapi/linearapi_test.go
+++ b/pkg/detectors/linearapi/linearapi_test.go
@@ -16,7 +16,7 @@ import (
func TestLinearAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/linemessaging/linemessaging_test.go b/pkg/detectors/linemessaging/linemessaging_test.go
index efad6188c..aa40011ab 100644
--- a/pkg/detectors/linemessaging/linemessaging_test.go
+++ b/pkg/detectors/linemessaging/linemessaging_test.go
@@ -16,7 +16,7 @@ import (
func TestLineMessaging_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/linenotify/linenotify_test.go b/pkg/detectors/linenotify/linenotify_test.go
index 2e1583fc8..0e08a75dd 100644
--- a/pkg/detectors/linenotify/linenotify_test.go
+++ b/pkg/detectors/linenotify/linenotify_test.go
@@ -16,7 +16,7 @@ import (
func TestLineNotify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/linkpreview/linkpreview_test.go b/pkg/detectors/linkpreview/linkpreview_test.go
index 9a2fd9dce..76bc42a78 100644
--- a/pkg/detectors/linkpreview/linkpreview_test.go
+++ b/pkg/detectors/linkpreview/linkpreview_test.go
@@ -16,7 +16,7 @@ import (
func TestLinkPreview_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/liveagent/liveagent_test.go b/pkg/detectors/liveagent/liveagent_test.go
index 705c15132..14084c4da 100644
--- a/pkg/detectors/liveagent/liveagent_test.go
+++ b/pkg/detectors/liveagent/liveagent_test.go
@@ -16,7 +16,7 @@ import (
func TestLiveAgent_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/lob/lob_test.go b/pkg/detectors/lob/lob_test.go
index e65fd9c8d..73cffba11 100644
--- a/pkg/detectors/lob/lob_test.go
+++ b/pkg/detectors/lob/lob_test.go
@@ -16,7 +16,7 @@ import (
func TestLob_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/locationiq/locationiq_test.go b/pkg/detectors/locationiq/locationiq_test.go
index 3504a64ab..368279972 100644
--- a/pkg/detectors/locationiq/locationiq_test.go
+++ b/pkg/detectors/locationiq/locationiq_test.go
@@ -16,7 +16,7 @@ import (
func TestLocationIQ_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/loginradius/loginradius_test.go b/pkg/detectors/loginradius/loginradius_test.go
index cbfe63956..1113da930 100644
--- a/pkg/detectors/loginradius/loginradius_test.go
+++ b/pkg/detectors/loginradius/loginradius_test.go
@@ -16,7 +16,7 @@ import (
func TestLoginradius_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/lokalisetoken/lokalisetoken_test.go b/pkg/detectors/lokalisetoken/lokalisetoken_test.go
index b95ccdd3a..58f0c3a2c 100644
--- a/pkg/detectors/lokalisetoken/lokalisetoken_test.go
+++ b/pkg/detectors/lokalisetoken/lokalisetoken_test.go
@@ -16,7 +16,7 @@ import (
func TestLokaliseToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/loyverse/loyverse_test.go b/pkg/detectors/loyverse/loyverse_test.go
index 21e1a41cd..b3d1b8de7 100644
--- a/pkg/detectors/loyverse/loyverse_test.go
+++ b/pkg/detectors/loyverse/loyverse_test.go
@@ -16,7 +16,7 @@ import (
func TestLoyverse_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/luno/luno_test.go b/pkg/detectors/luno/luno_test.go
index bd01f3410..4f5c8efc0 100644
--- a/pkg/detectors/luno/luno_test.go
+++ b/pkg/detectors/luno/luno_test.go
@@ -16,7 +16,7 @@ import (
func TestLuno_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/m3o/m3o_test.go b/pkg/detectors/m3o/m3o_test.go
index 288d82550..17afe1cc0 100644
--- a/pkg/detectors/m3o/m3o_test.go
+++ b/pkg/detectors/m3o/m3o_test.go
@@ -16,7 +16,7 @@ import (
func TestM3o_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/macaddress/macaddress_test.go b/pkg/detectors/macaddress/macaddress_test.go
index 95a1bd20f..2b6884cfc 100644
--- a/pkg/detectors/macaddress/macaddress_test.go
+++ b/pkg/detectors/macaddress/macaddress_test.go
@@ -16,7 +16,7 @@ import (
func TestMacaddress_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/magnetic/magnetic_test.go b/pkg/detectors/magnetic/magnetic_test.go
index 0698d5ff9..286417ff6 100644
--- a/pkg/detectors/magnetic/magnetic_test.go
+++ b/pkg/detectors/magnetic/magnetic_test.go
@@ -16,7 +16,7 @@ import (
func TestMagnetic_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailboxlayer/mailboxlayer_test.go b/pkg/detectors/mailboxlayer/mailboxlayer_test.go
index 56ecb1f73..80c748512 100644
--- a/pkg/detectors/mailboxlayer/mailboxlayer_test.go
+++ b/pkg/detectors/mailboxlayer/mailboxlayer_test.go
@@ -16,7 +16,7 @@ import (
func TestMailboxplayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailchimp/mailchimp.go b/pkg/detectors/mailchimp/mailchimp.go
new file mode 100644
index 000000000..2e1ec6131
--- /dev/null
+++ b/pkg/detectors/mailchimp/mailchimp.go
@@ -0,0 +1,76 @@
+package mailchimp
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ // TODO: Other country patterns?
+ keyPat = regexp.MustCompile(`[0-9a-f]{32}-us[0-9]{1,2}`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"-us"}
+}
+
+// FromData will find and optionally verify Mailchimp secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ //pretty standard regex match
+ matches := keyPat.FindAllString(dataStr, -1)
+
+ for _, match := range matches {
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Mailchimp,
+ Raw: []byte(match),
+ }
+
+ if verify {
+ datacenter := strings.Split(match, "-")[1]
+
+ client := common.RetryableHttpClient()
+ // https://mailchimp.com/developer/guides/marketing-api-conventions/
+ req, _ := http.NewRequest("GET", fmt.Sprintf("https://%s.api.mailchimp.com/3.0/", datacenter), nil)
+ req.SetBasicAuth("anystring", match)
+ res, err := client.Do(req)
+ if err != nil {
+ break
+ }
+ defer res.Body.Close()
+ if res.StatusCode == 200 {
+ s.Verified = true
+ } else {
+ s.Verified = false
+ }
+
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/mailchimp/mailchimp_test.go b/pkg/detectors/mailchimp/mailchimp_test.go
new file mode 100644
index 000000000..5bd0e9aa6
--- /dev/null
+++ b/pkg/detectors/mailchimp/mailchimp_test.go
@@ -0,0 +1,111 @@
+package mailchimp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestMailchimp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("MAILCHIMP")
+ secretInactive := testSecrets.MustGetField("MAILCHIMP_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a mailchimp secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Mailchimp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a mailchimp secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Mailchimp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Mailchimp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Mailchimp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/mailerlite/mailerlite_test.go b/pkg/detectors/mailerlite/mailerlite_test.go
index e6157cfa9..ee9fa8ba9 100644
--- a/pkg/detectors/mailerlite/mailerlite_test.go
+++ b/pkg/detectors/mailerlite/mailerlite_test.go
@@ -16,7 +16,7 @@ import (
func TestMailerlite_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailgun/mailgun_test.go b/pkg/detectors/mailgun/mailgun_test.go
index 1a6a95d57..6d105f13b 100644
--- a/pkg/detectors/mailgun/mailgun_test.go
+++ b/pkg/detectors/mailgun/mailgun_test.go
@@ -18,7 +18,7 @@ import (
func TestMailgun_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go b/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go
index 693d25ab9..f1ed0d68e 100644
--- a/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go
+++ b/pkg/detectors/mailjetbasicauth/mailjetbasicauth_test.go
@@ -16,7 +16,7 @@ import (
func TestMailJetBasicAuth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailjetsms/mailjetsms_test.go b/pkg/detectors/mailjetsms/mailjetsms_test.go
index 4aacce733..944b50480 100644
--- a/pkg/detectors/mailjetsms/mailjetsms_test.go
+++ b/pkg/detectors/mailjetsms/mailjetsms_test.go
@@ -16,7 +16,7 @@ import (
func TestMailJetSMS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailmodo/mailmodo_test.go b/pkg/detectors/mailmodo/mailmodo_test.go
index b76555b54..e3b3123f1 100644
--- a/pkg/detectors/mailmodo/mailmodo_test.go
+++ b/pkg/detectors/mailmodo/mailmodo_test.go
@@ -16,7 +16,7 @@ import (
func TestMailmodo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mailsac/mailsac_test.go b/pkg/detectors/mailsac/mailsac_test.go
index 8863d27a0..94c818958 100644
--- a/pkg/detectors/mailsac/mailsac_test.go
+++ b/pkg/detectors/mailsac/mailsac_test.go
@@ -16,7 +16,7 @@ import (
func TestMailsac_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mandrill/mandrill_test.go b/pkg/detectors/mandrill/mandrill_test.go
index 397ce58ea..43da79f76 100644
--- a/pkg/detectors/mandrill/mandrill_test.go
+++ b/pkg/detectors/mandrill/mandrill_test.go
@@ -16,7 +16,7 @@ import (
func TestMandrill_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/manifest/manifest_test.go b/pkg/detectors/manifest/manifest_test.go
index 258a752d8..95c920445 100644
--- a/pkg/detectors/manifest/manifest_test.go
+++ b/pkg/detectors/manifest/manifest_test.go
@@ -16,7 +16,7 @@ import (
func TestManifest_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mapbox/mapbox_test.go b/pkg/detectors/mapbox/mapbox_test.go
index 153f8825e..8a2c4bea8 100644
--- a/pkg/detectors/mapbox/mapbox_test.go
+++ b/pkg/detectors/mapbox/mapbox_test.go
@@ -16,7 +16,7 @@ import (
func TestMapBox_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mapquest/mapquest_test.go b/pkg/detectors/mapquest/mapquest_test.go
index fe1f3d83a..395b1ae40 100644
--- a/pkg/detectors/mapquest/mapquest_test.go
+++ b/pkg/detectors/mapquest/mapquest_test.go
@@ -16,7 +16,7 @@ import (
func TestMapquest_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/marketstack/marketstack_test.go b/pkg/detectors/marketstack/marketstack_test.go
index 44bab99f3..9b6babb68 100644
--- a/pkg/detectors/marketstack/marketstack_test.go
+++ b/pkg/detectors/marketstack/marketstack_test.go
@@ -16,7 +16,7 @@ import (
func TestMarketstack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go
index e961d410c..5704d3d4a 100644
--- a/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go
+++ b/pkg/detectors/mattermostpersonaltoken/mattermostpersonaltoken_test.go
@@ -16,7 +16,7 @@ import (
func TestMattermostPersonalToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mavenlink/mavenlink_test.go b/pkg/detectors/mavenlink/mavenlink_test.go
index e473f1ff3..8d4fc97f9 100644
--- a/pkg/detectors/mavenlink/mavenlink_test.go
+++ b/pkg/detectors/mavenlink/mavenlink_test.go
@@ -16,7 +16,7 @@ import (
func TestMavenlink_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/maxmindlicense/maxmindlicense_test.go b/pkg/detectors/maxmindlicense/maxmindlicense_test.go
index 995a051ef..e8af4c92a 100644
--- a/pkg/detectors/maxmindlicense/maxmindlicense_test.go
+++ b/pkg/detectors/maxmindlicense/maxmindlicense_test.go
@@ -16,7 +16,7 @@ import (
func TestMaxMindLicense_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/meaningcloud/meaningcloud_test.go b/pkg/detectors/meaningcloud/meaningcloud_test.go
index 31b076735..6f8105b5a 100644
--- a/pkg/detectors/meaningcloud/meaningcloud_test.go
+++ b/pkg/detectors/meaningcloud/meaningcloud_test.go
@@ -16,7 +16,7 @@ import (
func TestMeaningCloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mediastack/mediastack_test.go b/pkg/detectors/mediastack/mediastack_test.go
index 0ae98f1ed..81fa09cb0 100644
--- a/pkg/detectors/mediastack/mediastack_test.go
+++ b/pkg/detectors/mediastack/mediastack_test.go
@@ -16,7 +16,7 @@ import (
func TestMediaStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/meistertask/meistertask_test.go b/pkg/detectors/meistertask/meistertask_test.go
index c537e64b6..e7f71ecc7 100644
--- a/pkg/detectors/meistertask/meistertask_test.go
+++ b/pkg/detectors/meistertask/meistertask_test.go
@@ -16,7 +16,7 @@ import (
func TestMeistertask_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mesibo/mesibo_test.go b/pkg/detectors/mesibo/mesibo_test.go
index 2f33c22db..59097868c 100644
--- a/pkg/detectors/mesibo/mesibo_test.go
+++ b/pkg/detectors/mesibo/mesibo_test.go
@@ -16,7 +16,7 @@ import (
func TestMesibo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/messagebird/messagebird_test.go b/pkg/detectors/messagebird/messagebird_test.go
index 0e2467634..928eac459 100644
--- a/pkg/detectors/messagebird/messagebird_test.go
+++ b/pkg/detectors/messagebird/messagebird_test.go
@@ -16,7 +16,7 @@ import (
func TestMessageBird_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/metrilo/metrilo_test.go b/pkg/detectors/metrilo/metrilo_test.go
index 6bcf086b6..cbaf30194 100644
--- a/pkg/detectors/metrilo/metrilo_test.go
+++ b/pkg/detectors/metrilo/metrilo_test.go
@@ -16,7 +16,7 @@ import (
func TestMetrilo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go
index db4b53c6a..79ea4dbd5 100644
--- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go
+++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook_test.go
@@ -16,7 +16,7 @@ import (
func TestMicrosoftTeamsWebhook_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mindmeister/mindmeister_test.go b/pkg/detectors/mindmeister/mindmeister_test.go
index ce66487a5..74db9627c 100644
--- a/pkg/detectors/mindmeister/mindmeister_test.go
+++ b/pkg/detectors/mindmeister/mindmeister_test.go
@@ -16,7 +16,7 @@ import (
func TestMindmeister_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mite/mite_test.go b/pkg/detectors/mite/mite_test.go
index 836b0f9bb..a4959bf11 100644
--- a/pkg/detectors/mite/mite_test.go
+++ b/pkg/detectors/mite/mite_test.go
@@ -16,7 +16,7 @@ import (
func TestMite_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mixmax/mixmax_test.go b/pkg/detectors/mixmax/mixmax_test.go
index 065da4dce..a97f1cd67 100644
--- a/pkg/detectors/mixmax/mixmax_test.go
+++ b/pkg/detectors/mixmax/mixmax_test.go
@@ -16,7 +16,7 @@ import (
func TestMixmax_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mixpanel/mixpanel_test.go b/pkg/detectors/mixpanel/mixpanel_test.go
index 3c496849d..e70c56e3e 100644
--- a/pkg/detectors/mixpanel/mixpanel_test.go
+++ b/pkg/detectors/mixpanel/mixpanel_test.go
@@ -16,7 +16,7 @@ import (
func TestMixpanel_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/monday/monday_test.go b/pkg/detectors/monday/monday_test.go
index 7c0d91600..b12e425eb 100644
--- a/pkg/detectors/monday/monday_test.go
+++ b/pkg/detectors/monday/monday_test.go
@@ -16,7 +16,7 @@ import (
func TestMonday_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/moonclerk/moonclerk.go b/pkg/detectors/moonclerk/moonclerk.go
new file mode 100644
index 000000000..07876807b
--- /dev/null
+++ b/pkg/detectors/moonclerk/moonclerk.go
@@ -0,0 +1,74 @@
+package moonclerk
+
+import (
+ "context"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"moonclerk"}) + `\b([0-9a-z]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"moonclerk"}
+}
+
+// FromData will find and optionally verify Moonclerk secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Moonclerk,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://api.moonclerk.com/forms", nil)
+ req.Header.Add("Accept", "application/vnd.moonclerk+json;version=1")
+ req.Header.Add("Authorization", "Token token="+resMatch)
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/moonclerk/moonclerk_test.go b/pkg/detectors/moonclerk/moonclerk_test.go
new file mode 100644
index 000000000..9a6caaffc
--- /dev/null
+++ b/pkg/detectors/moonclerk/moonclerk_test.go
@@ -0,0 +1,113 @@
+package moonclerk
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestMoonclerk_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("MOONCLERCK")
+ inactiveSecret := testSecrets.MustGetField("MOONCLERCK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a moonclerk secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Moonclerk,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a moonclerk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Moonclerk,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Moonclerk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Moonclerk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/moosend/moosend_test.go b/pkg/detectors/moosend/moosend_test.go
index c1a6b4a54..b8d1f43b0 100644
--- a/pkg/detectors/moosend/moosend_test.go
+++ b/pkg/detectors/moosend/moosend_test.go
@@ -16,7 +16,7 @@ import (
func TestMoosend_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/mrticktock/mrticktock_test.go b/pkg/detectors/mrticktock/mrticktock_test.go
index 1bd1fe064..4d3e1ed40 100644
--- a/pkg/detectors/mrticktock/mrticktock_test.go
+++ b/pkg/detectors/mrticktock/mrticktock_test.go
@@ -16,7 +16,7 @@ import (
func TestMrticktock_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/myfreshworks/myfreshworks_test.go b/pkg/detectors/myfreshworks/myfreshworks_test.go
index 7f4b162fe..20d0a3479 100644
--- a/pkg/detectors/myfreshworks/myfreshworks_test.go
+++ b/pkg/detectors/myfreshworks/myfreshworks_test.go
@@ -16,7 +16,7 @@ import (
func TestMyfreshworks_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/myintervals/myintervals_test.go b/pkg/detectors/myintervals/myintervals_test.go
index e1fd16a6f..6c560c3f2 100644
--- a/pkg/detectors/myintervals/myintervals_test.go
+++ b/pkg/detectors/myintervals/myintervals_test.go
@@ -16,7 +16,7 @@ import (
func TestMyintervals_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go b/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go
index e53973f70..cad4ab278 100644
--- a/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go
+++ b/pkg/detectors/nasdaqdatalink/nasdaqdatalink_test.go
@@ -16,7 +16,7 @@ import (
func TestNasdaqDataLink_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nethunt/nethunt_test.go b/pkg/detectors/nethunt/nethunt_test.go
index 1d2182cc3..efc864dc2 100644
--- a/pkg/detectors/nethunt/nethunt_test.go
+++ b/pkg/detectors/nethunt/nethunt_test.go
@@ -16,7 +16,7 @@ import (
func TestNethunt_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/netlify/netlify_test.go b/pkg/detectors/netlify/netlify_test.go
index 8b8c535aa..2366eb43e 100644
--- a/pkg/detectors/netlify/netlify_test.go
+++ b/pkg/detectors/netlify/netlify_test.go
@@ -16,7 +16,7 @@ import (
func TestNetlify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/neutrinoapi/neutrinoapi_test.go b/pkg/detectors/neutrinoapi/neutrinoapi_test.go
index 07de39413..bd2f01925 100644
--- a/pkg/detectors/neutrinoapi/neutrinoapi_test.go
+++ b/pkg/detectors/neutrinoapi/neutrinoapi_test.go
@@ -16,7 +16,7 @@ import (
func TestNeutrinoApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go b/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go
index 2c1f1a24b..a61e1719c 100644
--- a/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go
+++ b/pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestNewRelicPersonalApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/newsapi/newsapi_test.go b/pkg/detectors/newsapi/newsapi_test.go
index 21bec5b44..19b00a2bc 100644
--- a/pkg/detectors/newsapi/newsapi_test.go
+++ b/pkg/detectors/newsapi/newsapi_test.go
@@ -16,7 +16,7 @@ import (
func TestNewsapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/newscatcher/newscatcher_test.go b/pkg/detectors/newscatcher/newscatcher_test.go
index 243a9b663..0f1acfbfb 100644
--- a/pkg/detectors/newscatcher/newscatcher_test.go
+++ b/pkg/detectors/newscatcher/newscatcher_test.go
@@ -16,7 +16,7 @@ import (
func TestNewscatcher_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nexmoapikey/nexmoapikey_test.go b/pkg/detectors/nexmoapikey/nexmoapikey_test.go
index 1bebafc96..1cb369214 100644
--- a/pkg/detectors/nexmoapikey/nexmoapikey_test.go
+++ b/pkg/detectors/nexmoapikey/nexmoapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestNexmoApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nftport/nftport_test.go b/pkg/detectors/nftport/nftport_test.go
index f2e12a779..e2f7e3ae6 100644
--- a/pkg/detectors/nftport/nftport_test.go
+++ b/pkg/detectors/nftport/nftport_test.go
@@ -16,7 +16,7 @@ import (
func TestNftport_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nicereply/nicereply_test.go b/pkg/detectors/nicereply/nicereply_test.go
index 35bdd9090..0a74f233d 100644
--- a/pkg/detectors/nicereply/nicereply_test.go
+++ b/pkg/detectors/nicereply/nicereply_test.go
@@ -16,7 +16,7 @@ import (
func TestNicereply_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nimble/nimble_test.go b/pkg/detectors/nimble/nimble_test.go
index 0b68ed82d..b14f0a944 100644
--- a/pkg/detectors/nimble/nimble_test.go
+++ b/pkg/detectors/nimble/nimble_test.go
@@ -16,7 +16,7 @@ import (
func TestNimble_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nitro/nitro_test.go b/pkg/detectors/nitro/nitro_test.go
index b6b891867..40b714577 100644
--- a/pkg/detectors/nitro/nitro_test.go
+++ b/pkg/detectors/nitro/nitro_test.go
@@ -16,7 +16,7 @@ import (
func TestNitro_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/noticeable/noticeable_test.go b/pkg/detectors/noticeable/noticeable_test.go
index 6c4fe8f37..5e2f03810 100644
--- a/pkg/detectors/noticeable/noticeable_test.go
+++ b/pkg/detectors/noticeable/noticeable_test.go
@@ -16,7 +16,7 @@ import (
func TestNoticeable_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/notion/notion_test.go b/pkg/detectors/notion/notion_test.go
index deb1ae217..1818796f5 100644
--- a/pkg/detectors/notion/notion_test.go
+++ b/pkg/detectors/notion/notion_test.go
@@ -16,7 +16,7 @@ import (
func TestNotion_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/numverify/numverify_test.go b/pkg/detectors/numverify/numverify_test.go
index c73d916a9..3f1d4ee8a 100644
--- a/pkg/detectors/numverify/numverify_test.go
+++ b/pkg/detectors/numverify/numverify_test.go
@@ -16,7 +16,7 @@ import (
func TestNumverify_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nutritionix/nutritionix_test.go b/pkg/detectors/nutritionix/nutritionix_test.go
index ddb49d9ff..fb280086d 100644
--- a/pkg/detectors/nutritionix/nutritionix_test.go
+++ b/pkg/detectors/nutritionix/nutritionix_test.go
@@ -16,7 +16,7 @@ import (
func TestNutritionix_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nylas/nylas_test.go b/pkg/detectors/nylas/nylas_test.go
index 439ea16f7..c101e9811 100644
--- a/pkg/detectors/nylas/nylas_test.go
+++ b/pkg/detectors/nylas/nylas_test.go
@@ -16,7 +16,7 @@ import (
func TestNylas_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/nytimes/nytimes_test.go b/pkg/detectors/nytimes/nytimes_test.go
index 8e789d18d..85f12de32 100644
--- a/pkg/detectors/nytimes/nytimes_test.go
+++ b/pkg/detectors/nytimes/nytimes_test.go
@@ -16,7 +16,7 @@ import (
func TestNytimes_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/oanda/oanda_test.go b/pkg/detectors/oanda/oanda_test.go
index 31c15e9c4..656768d2c 100644
--- a/pkg/detectors/oanda/oanda_test.go
+++ b/pkg/detectors/oanda/oanda_test.go
@@ -16,7 +16,7 @@ import (
func TestOanda_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go
new file mode 100644
index 000000000..625592b95
--- /dev/null
+++ b/pkg/detectors/okta/okta.go
@@ -0,0 +1,82 @@
+package okta
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
+ tokenPat = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
+ // TODO: Oauth client secrets
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"okta"}
+}
+
+// FromData will find and optionally verify Okta secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ for _, tokenMatch := range tokenPat.FindAll(data, -1) {
+ token := string(tokenMatch)
+
+ for _, domainMatch := range domainPat.FindAll(data, -1) {
+ domain := string(domainMatch)
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Okta,
+ Raw: []byte(token),
+ }
+
+ if verify {
+ // curl -v -X GET \
+ // -H "Accept: application/json" \
+ // -H "Content-Type: application/json" \
+ // -H "Authorization: Bearer token" \
+ // "https://subdomain.okta.com/api/v1/groups?limit=1"
+ //
+
+ url := fmt.Sprintf("https://%s/api/v1/groups?limit=1", domain)
+ req, err := http.NewRequest("GET", url, nil)
+ if err != nil {
+ return results, err
+ }
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token))
+
+ resp, err := common.RetryableHttpClient().Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+ }
+
+ return
+}
diff --git a/pkg/detectors/okta/okta_test.go b/pkg/detectors/okta/okta_test.go
new file mode 100644
index 000000000..e52970d7a
--- /dev/null
+++ b/pkg/detectors/okta/okta_test.go
@@ -0,0 +1,113 @@
+package okta
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestOkta_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("OKTA")
+ secretInactive := testSecrets.MustGetField("OKTA_INACTIVE")
+ domain := testSecrets.MustGetField("OKTA_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found token, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a okta secret %s within oktaDomain %s", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Okta,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a okta secret %s within oktaDomain %s", secretInactive, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Okta,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Okta.FromData) error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Okta.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/omnisend/omnisend_test.go b/pkg/detectors/omnisend/omnisend_test.go
index 877bfa2a7..5d42751f1 100644
--- a/pkg/detectors/omnisend/omnisend_test.go
+++ b/pkg/detectors/omnisend/omnisend_test.go
@@ -16,7 +16,7 @@ import (
func TestOmnisend_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/onedesk/onedesk_test.go b/pkg/detectors/onedesk/onedesk_test.go
index 1eaabe729..e077b8ea2 100644
--- a/pkg/detectors/onedesk/onedesk_test.go
+++ b/pkg/detectors/onedesk/onedesk_test.go
@@ -16,7 +16,7 @@ import (
func TestOnedesk_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.go
new file mode 100644
index 000000000..87d8a2edc
--- /dev/null
+++ b/pkg/detectors/onelogin/onelogin.go
@@ -0,0 +1,87 @@
+package onelogin
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ oauthClientIDPat = regexp.MustCompile(`(?i)id[a-zA-Z0-9_' "=]{0,20}([a-z0-9]{64})`)
+ oauthClientSecretPat = regexp.MustCompile(`(?i)secret[a-zA-Z0-9_' "=]{0,20}([a-z0-9]{64})`)
+
+ // TODO: Legacy API tokens
+
+ falsePositives = []string{"example"}
+
+ apiDomains = []string{"api.us.onelogin.com", "api.eu.onelogin.com"}
+
+ client = http.Client{Timeout: time.Second * 5}
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"onelogin"}
+}
+
+// FromData will find and optionally verify Onelogin secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ for _, clientID := range oauthClientIDPat.FindAllStringSubmatch(dataStr, -1) {
+ if len(clientID) != 2 {
+ continue
+ }
+ for _, clientSecret := range oauthClientSecretPat.FindAllStringSubmatch(dataStr, -1) {
+ if len(clientSecret) != 2 {
+ continue
+ }
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_OneLogin,
+ Raw: []byte(clientID[1]),
+ Redacted: clientID[1],
+ }
+
+ if verify {
+ for _, domain := range apiDomains {
+ tokenURL := fmt.Sprintf("https://%s/auth/oauth2/v2/token", domain)
+ req, _ := http.NewRequestWithContext(ctx, "POST", tokenURL, strings.NewReader(`{"grant_type":"client_credentials"}`))
+ req.Header.Add("Authorization", fmt.Sprintf("client_id:%s, client_secret:%s", clientID[1], clientSecret[1]))
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s.Verified = true
+ break
+ }
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+ }
+
+ return
+}
diff --git a/pkg/detectors/onelogin/onelogin_test.go b/pkg/detectors/onelogin/onelogin_test.go
new file mode 100644
index 000000000..2c4d391db
--- /dev/null
+++ b/pkg/detectors/onelogin/onelogin_test.go
@@ -0,0 +1,112 @@
+package onelogin
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestOnelogin_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ONELOGIN")
+ secretInactive := testSecrets.MustGetField("ONELOGIN_INACTIVE")
+ id := testSecrets.MustGetField("ONELOGIN_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a onelogin secret=%s within onelogin id %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_OneLogin,
+ Verified: true,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a onelogin secret=%s within onelogin id %s", secretInactive, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_OneLogin,
+ Verified: false,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Onelogin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Onelogin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/onepagecrm/onepagecrm_test.go b/pkg/detectors/onepagecrm/onepagecrm_test.go
index 119844888..055f1b8e6 100644
--- a/pkg/detectors/onepagecrm/onepagecrm_test.go
+++ b/pkg/detectors/onepagecrm/onepagecrm_test.go
@@ -16,7 +16,7 @@ import (
func TestOnepageCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/onwaterio/onwaterio_test.go b/pkg/detectors/onwaterio/onwaterio_test.go
index 3b420d55d..543615109 100644
--- a/pkg/detectors/onwaterio/onwaterio_test.go
+++ b/pkg/detectors/onwaterio/onwaterio_test.go
@@ -16,7 +16,7 @@ import (
func TestOnWaterIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/oopspam/oopspam_test.go b/pkg/detectors/oopspam/oopspam_test.go
index 1b5b75ece..d427fbc62 100644
--- a/pkg/detectors/oopspam/oopspam_test.go
+++ b/pkg/detectors/oopspam/oopspam_test.go
@@ -16,7 +16,7 @@ import (
func TestOOPSpam_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/opencagedata/opencagedata_test.go b/pkg/detectors/opencagedata/opencagedata_test.go
index a7db94e70..01b9e72f4 100644
--- a/pkg/detectors/opencagedata/opencagedata_test.go
+++ b/pkg/detectors/opencagedata/opencagedata_test.go
@@ -16,7 +16,7 @@ import (
func TestOpenCageData_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/opengraphr/opengraphr_test.go b/pkg/detectors/opengraphr/opengraphr_test.go
index ef47ca901..6e2104a75 100644
--- a/pkg/detectors/opengraphr/opengraphr_test.go
+++ b/pkg/detectors/opengraphr/opengraphr_test.go
@@ -16,7 +16,7 @@ import (
func TestOpengraphr_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/openuv/openuv_test.go b/pkg/detectors/openuv/openuv_test.go
index 0b997af30..e431805c5 100644
--- a/pkg/detectors/openuv/openuv_test.go
+++ b/pkg/detectors/openuv/openuv_test.go
@@ -16,7 +16,7 @@ import (
func TestOpenuv_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/openweather/openweather_test.go b/pkg/detectors/openweather/openweather_test.go
index 0ca781cb9..63428a00c 100644
--- a/pkg/detectors/openweather/openweather_test.go
+++ b/pkg/detectors/openweather/openweather_test.go
@@ -16,7 +16,7 @@ import (
func TestOpenWeather_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/optimizely/optimizely_test.go b/pkg/detectors/optimizely/optimizely_test.go
index 782c49315..84973f295 100644
--- a/pkg/detectors/optimizely/optimizely_test.go
+++ b/pkg/detectors/optimizely/optimizely_test.go
@@ -16,7 +16,7 @@ import (
func TestOptimizely_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/owlbot/owlbot_test.go b/pkg/detectors/owlbot/owlbot_test.go
index 99e9ca997..212ee321f 100644
--- a/pkg/detectors/owlbot/owlbot_test.go
+++ b/pkg/detectors/owlbot/owlbot_test.go
@@ -16,7 +16,7 @@ import (
func TestOwlbot_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go
index 21f5d0128..83d8195d0 100644
--- a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go
+++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestPagerDutyApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pandadoc/pandadoc_test.go b/pkg/detectors/pandadoc/pandadoc_test.go
index c3d1ec952..5b6fefd8a 100644
--- a/pkg/detectors/pandadoc/pandadoc_test.go
+++ b/pkg/detectors/pandadoc/pandadoc_test.go
@@ -16,7 +16,7 @@ import (
func TestPandadoc_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/paralleldots/paralleldots_test.go b/pkg/detectors/paralleldots/paralleldots_test.go
index 966ce6a42..8f826759d 100644
--- a/pkg/detectors/paralleldots/paralleldots_test.go
+++ b/pkg/detectors/paralleldots/paralleldots_test.go
@@ -16,7 +16,7 @@ import (
func TestParalleldots_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/partnerstack/partnerstack_test.go b/pkg/detectors/partnerstack/partnerstack_test.go
index 363984bbd..c2790b2e6 100644
--- a/pkg/detectors/partnerstack/partnerstack_test.go
+++ b/pkg/detectors/partnerstack/partnerstack_test.go
@@ -16,7 +16,7 @@ import (
func TestPartnerstack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/passbase/passbase_test.go b/pkg/detectors/passbase/passbase_test.go
index 9ed58a0ef..4ff11b18a 100644
--- a/pkg/detectors/passbase/passbase_test.go
+++ b/pkg/detectors/passbase/passbase_test.go
@@ -16,7 +16,7 @@ import (
func TestPassbase_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pastebin/pastebin_test.go b/pkg/detectors/pastebin/pastebin_test.go
index 6479ede7f..3a44f913a 100644
--- a/pkg/detectors/pastebin/pastebin_test.go
+++ b/pkg/detectors/pastebin/pastebin_test.go
@@ -16,7 +16,7 @@ import (
func TestPastebin_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/paymoapp/paymoapp_test.go b/pkg/detectors/paymoapp/paymoapp_test.go
index f306aa457..f1ed16c7e 100644
--- a/pkg/detectors/paymoapp/paymoapp_test.go
+++ b/pkg/detectors/paymoapp/paymoapp_test.go
@@ -16,7 +16,7 @@ import (
func TestPaymoapp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/paymongo/paymongo.go b/pkg/detectors/paymongo/paymongo.go
new file mode 100644
index 000000000..c6cc92ed6
--- /dev/null
+++ b/pkg/detectors/paymongo/paymongo.go
@@ -0,0 +1,75 @@
+package paymongo
+
+import (
+ "context"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"paymongo"}) + `\b([a-zA-Z0-9_]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"paymongo"}
+}
+
+// FromData will find and optionally verify Paymongo secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Paymongo,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ payload := strings.NewReader(`{"data":{"attributes":{"type":"paymaya"}}}`)
+ req, _ := http.NewRequest("POST", "https://api.paymongo.com/v1/payment_methods", payload)
+ req.Header.Add("Content-Type", "application/json")
+ req.SetBasicAuth(resMatch, "")
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/paymongo/paymongo_test.go b/pkg/detectors/paymongo/paymongo_test.go
new file mode 100644
index 000000000..c607e11ed
--- /dev/null
+++ b/pkg/detectors/paymongo/paymongo_test.go
@@ -0,0 +1,113 @@
+package paymongo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestPaymongo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAYMONGO")
+ inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paymongo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paymongo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paypaloauth/paypaloauth_test.go b/pkg/detectors/paypaloauth/paypaloauth_test.go
index ee8704333..468815ca6 100644
--- a/pkg/detectors/paypaloauth/paypaloauth_test.go
+++ b/pkg/detectors/paypaloauth/paypaloauth_test.go
@@ -16,7 +16,7 @@ import (
func TestPaypalOauth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/paystack/paystack_test.go b/pkg/detectors/paystack/paystack_test.go
index de08aacd2..beded3b50 100644
--- a/pkg/detectors/paystack/paystack_test.go
+++ b/pkg/detectors/paystack/paystack_test.go
@@ -16,7 +16,7 @@ import (
func TestPaystack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pdflayer/pdflayer_test.go b/pkg/detectors/pdflayer/pdflayer_test.go
index cbbf9d3eb..100f40d64 100644
--- a/pkg/detectors/pdflayer/pdflayer_test.go
+++ b/pkg/detectors/pdflayer/pdflayer_test.go
@@ -16,7 +16,7 @@ import (
func TestPdfLayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pdfshift/pdfshift_test.go b/pkg/detectors/pdfshift/pdfshift_test.go
index 009fdbe16..c56923401 100644
--- a/pkg/detectors/pdfshift/pdfshift_test.go
+++ b/pkg/detectors/pdfshift/pdfshift_test.go
@@ -16,7 +16,7 @@ import (
func TestPdfShift_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/peopledatalabs/peopledatalabs_test.go b/pkg/detectors/peopledatalabs/peopledatalabs_test.go
index a43a37e12..fe96d3d2f 100644
--- a/pkg/detectors/peopledatalabs/peopledatalabs_test.go
+++ b/pkg/detectors/peopledatalabs/peopledatalabs_test.go
@@ -16,7 +16,7 @@ import (
func TestPeopleDataLabs_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pepipost/pepipost_test.go b/pkg/detectors/pepipost/pepipost_test.go
index 5541196fd..5d77261b2 100644
--- a/pkg/detectors/pepipost/pepipost_test.go
+++ b/pkg/detectors/pepipost/pepipost_test.go
@@ -16,7 +16,7 @@ import (
func TestPepipost_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pipedrive/pipedrive_test.go b/pkg/detectors/pipedrive/pipedrive_test.go
index 090cb8a39..eca4fe0de 100644
--- a/pkg/detectors/pipedrive/pipedrive_test.go
+++ b/pkg/detectors/pipedrive/pipedrive_test.go
@@ -16,7 +16,7 @@ import (
func TestPipedrive_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pivotaltracker/pivotaltracker.go b/pkg/detectors/pivotaltracker/pivotaltracker.go
new file mode 100644
index 000000000..6a86c268f
--- /dev/null
+++ b/pkg/detectors/pivotaltracker/pivotaltracker.go
@@ -0,0 +1,77 @@
+package pivotaltracker
+
+import (
+ "context"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ // Get token at https://www.pivotaltracker.com/profile
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pivotal"}) + `([a-z0-9]{32})`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"pivotal"}
+}
+
+// FromData will find and optionally verify PivotalTracker secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+
+ // First match is entire regex, second is the first group.
+ if len(match) != 2 {
+ continue
+ }
+
+ token := match[1]
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_PivotalTracker,
+ Raw: []byte(token),
+ }
+
+ if verify {
+ client := common.RetryableHttpClient()
+ // https://www.pivotaltracker.com/help/api/rest/v5#top
+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://www.pivotaltracker.com/services/v5/me", nil)
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ req.Header.Add("X-TrackerToken", token)
+ res, err := client.Do(req)
+ if err != nil {
+ break
+ }
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s.Verified = true
+ }
+
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/pivotaltracker/pivotaltracker_test.go b/pkg/detectors/pivotaltracker/pivotaltracker_test.go
new file mode 100644
index 000000000..dd2c9f50c
--- /dev/null
+++ b/pkg/detectors/pivotaltracker/pivotaltracker_test.go
@@ -0,0 +1,112 @@
+package pivotaltracker
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestPivotalTracker_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PIVOTALTRACKER")
+ secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PivotalTracker,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PivotalTracker,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pixabay/pixabay_test.go b/pkg/detectors/pixabay/pixabay_test.go
index 293f6c5f4..185ae0600 100644
--- a/pkg/detectors/pixabay/pixabay_test.go
+++ b/pkg/detectors/pixabay/pixabay_test.go
@@ -16,7 +16,7 @@ import (
func TestPixabay_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/plaidkey/plaidkey_test.go b/pkg/detectors/plaidkey/plaidkey_test.go
index 2e4654563..1904ff0b7 100644
--- a/pkg/detectors/plaidkey/plaidkey_test.go
+++ b/pkg/detectors/plaidkey/plaidkey_test.go
@@ -16,7 +16,7 @@ import (
func TestPlaidKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/planyo/planyo_test.go b/pkg/detectors/planyo/planyo_test.go
index 1dd4b3475..096ae1c15 100644
--- a/pkg/detectors/planyo/planyo_test.go
+++ b/pkg/detectors/planyo/planyo_test.go
@@ -16,7 +16,7 @@ import (
func TestPlanyo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/plivo/plivo_test.go b/pkg/detectors/plivo/plivo_test.go
index a669ff186..c62d0034c 100644
--- a/pkg/detectors/plivo/plivo_test.go
+++ b/pkg/detectors/plivo/plivo_test.go
@@ -16,7 +16,7 @@ import (
func TestPlivo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/poloniex/poloniex_test.go b/pkg/detectors/poloniex/poloniex_test.go
index 28886711d..6068e53c4 100644
--- a/pkg/detectors/poloniex/poloniex_test.go
+++ b/pkg/detectors/poloniex/poloniex_test.go
@@ -16,7 +16,7 @@ import (
func TestPoloniex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/polygon/polygon.go b/pkg/detectors/polygon/polygon.go
new file mode 100644
index 000000000..63c251806
--- /dev/null
+++ b/pkg/detectors/polygon/polygon.go
@@ -0,0 +1,73 @@
+package polygon
+
+import (
+ "context"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"polygon"}) + `\b([a-z0-9A-Z]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"polygon"}
+}
+
+// FromData will find and optionally verify Polygon secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Polygon,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://api.polygon.io/v2/reference/locales?apiKey="+resMatch, nil)
+
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/polygon/polygon_test.go b/pkg/detectors/polygon/polygon_test.go
new file mode 100644
index 000000000..73d5df286
--- /dev/null
+++ b/pkg/detectors/polygon/polygon_test.go
@@ -0,0 +1,113 @@
+package polygon
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestPolygon_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POLYGON_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Polygon,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Polygon,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/positionstack/positionstack_test.go b/pkg/detectors/positionstack/positionstack_test.go
index 9de12b5c9..be5edbe23 100644
--- a/pkg/detectors/positionstack/positionstack_test.go
+++ b/pkg/detectors/positionstack/positionstack_test.go
@@ -16,7 +16,7 @@ import (
func TestPositionStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/postageapp/postageapp_test.go b/pkg/detectors/postageapp/postageapp_test.go
index ce36e75fe..ff4c0f97b 100644
--- a/pkg/detectors/postageapp/postageapp_test.go
+++ b/pkg/detectors/postageapp/postageapp_test.go
@@ -16,7 +16,7 @@ import (
func TestPostageApp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/posthog/posthog_test.go b/pkg/detectors/posthog/posthog_test.go
index 6f80cd0de..9baac55c0 100644
--- a/pkg/detectors/posthog/posthog_test.go
+++ b/pkg/detectors/posthog/posthog_test.go
@@ -16,7 +16,7 @@ import (
func TestAppPosthog_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/postman/postman_test.go b/pkg/detectors/postman/postman_test.go
index aa9e2ecd2..4efcc3403 100644
--- a/pkg/detectors/postman/postman_test.go
+++ b/pkg/detectors/postman/postman_test.go
@@ -16,7 +16,7 @@ import (
func TestPostman_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/postmark/postmark_test.go b/pkg/detectors/postmark/postmark_test.go
index 025d91518..188876790 100644
--- a/pkg/detectors/postmark/postmark_test.go
+++ b/pkg/detectors/postmark/postmark_test.go
@@ -16,7 +16,7 @@ import (
func TestPostmark_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/powrbot/powrbot_test.go b/pkg/detectors/powrbot/powrbot_test.go
index a76567496..cbe38f9ad 100644
--- a/pkg/detectors/powrbot/powrbot_test.go
+++ b/pkg/detectors/powrbot/powrbot_test.go
@@ -16,7 +16,7 @@ import (
func TestPowrbot_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/privatekey/cracker.go b/pkg/detectors/privatekey/cracker.go
new file mode 100644
index 000000000..80b4cf6f3
--- /dev/null
+++ b/pkg/detectors/privatekey/cracker.go
@@ -0,0 +1,37 @@
+package privatekey
+
+import (
+ "bytes"
+ "crypto/x509"
+ _ "embed"
+ "errors"
+
+ "golang.org/x/crypto/ssh"
+)
+
+//go:embed "rockyou-15.txt"
+var rawCrackList []byte
+var passphrases [][]byte
+
+func init() {
+ passphrases = bytes.Split(rawCrackList, []byte("\n"))
+}
+
+var (
+ ErrUncrackable = errors.New("unable to crack encryption")
+)
+
+func crack(in []byte) (interface{}, error) {
+ for _, passphrase := range passphrases {
+ parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
+ if err != nil {
+ if errors.Is(err, x509.IncorrectPasswordError) {
+ continue
+ } else {
+ return nil, err
+ }
+ }
+ return parsed, nil
+ }
+ return nil, ErrUncrackable
+}
diff --git a/pkg/detectors/privatekey/cracker_test.go b/pkg/detectors/privatekey/cracker_test.go
new file mode 100644
index 000000000..196d87c1a
--- /dev/null
+++ b/pkg/detectors/privatekey/cracker_test.go
@@ -0,0 +1,93 @@
+package privatekey
+
+import (
+ _ "embed"
+ "testing"
+
+ "golang.org/x/crypto/ssh"
+)
+
+var (
+ testEncryptedKeyCorrectPassword = []byte("123456")
+ testEncryptedKeyIncorrectPassword = []byte("incorrect")
+ testEncryptedKey = []byte(normalize(string(`-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
+xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
+utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
+inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
+OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
+sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
+NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
+WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
+lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
+UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
+uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
+Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
+SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
+kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
+wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
+TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
+nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
+TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
+8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
+wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
+akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
+ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
+Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
+e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
+s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
+Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
+kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
+Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
+fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
+S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
+sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
+c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
+/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
+muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
+iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
+f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
+Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
+-----END OPENSSH PRIVATE KEY-----`)))
+)
+
+func Test_crack(t *testing.T) {
+ tests := []struct {
+ name string
+ in []byte
+ wantErr bool
+ }{
+ {
+ name: "crackable",
+ wantErr: false,
+ in: testEncryptedKey,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := crack(tt.in)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ })
+ }
+}
+
+func BenchmarkParseWrongPassword(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ ssh.ParseRawPrivateKeyWithPassphrase(testEncryptedKey, testEncryptedKeyIncorrectPassword)
+ }
+}
+
+func BenchmarkParseRightPassword(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ ssh.ParseRawPrivateKeyWithPassphrase(testEncryptedKey, testEncryptedKeyCorrectPassword)
+ }
+}
+
+func BenchmarkCracker(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ crack(testEncryptedKey)
+ }
+}
diff --git a/pkg/detectors/privatekey/fingerprint.go b/pkg/detectors/privatekey/fingerprint.go
new file mode 100644
index 000000000..afb24f5d0
--- /dev/null
+++ b/pkg/detectors/privatekey/fingerprint.go
@@ -0,0 +1,55 @@
+package privatekey
+
+import (
+ "crypto/ecdsa"
+ "crypto/ed25519"
+ "crypto/rsa"
+ "crypto/sha1"
+ "crypto/x509"
+ "encoding/hex"
+ "errors"
+ "strings"
+
+ "golang.org/x/crypto/ssh"
+)
+
+var (
+ ErrNotSupported = errors.New("key type not supported")
+ ErrEncryptedKey = errors.New("key is encrypted")
+)
+
+func FingerprintPEMKey(in []byte) (string, error) {
+ parsedKey, err := ssh.ParseRawPrivateKey(in)
+ if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
+ parsedKey, err = crack(in)
+ if err != nil {
+ return "", err
+ }
+ } else if err != nil {
+ return "", err
+ }
+
+ var pubKey interface{}
+ switch privateKey := parsedKey.(type) {
+ case *rsa.PrivateKey:
+ pubKey = &privateKey.PublicKey
+ case *ecdsa.PrivateKey:
+ pubKey = &privateKey.PublicKey
+ case ed25519.PrivateKey:
+ pubKey = privateKey.Public()
+ // No fingerprinting support for DSA
+ // case *dsa.PrivateKey:
+ // pubKey = privateKey.PublicKey
+ default:
+ return "", ErrNotSupported
+ }
+
+ publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
+ if err != nil {
+ return "", err
+ }
+
+ publicKeyFingerprint := sha1.Sum(publickeyBytes)
+ publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
+ return publicKeyFingerprintInHex, nil
+}
diff --git a/pkg/detectors/privatekey/normalize.go b/pkg/detectors/privatekey/normalize.go
new file mode 100644
index 000000000..cc8df87d0
--- /dev/null
+++ b/pkg/detectors/privatekey/normalize.go
@@ -0,0 +1,33 @@
+package privatekey
+
+import (
+ "strings"
+)
+
+const b64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
+
+func normalize(in string) string {
+ in = strings.ReplaceAll(in, `"`, "")
+ in = strings.ReplaceAll(in, `'`, "")
+ in = strings.ReplaceAll(in, "\t", "")
+ in = strings.ReplaceAll(in, `\t`, "")
+ in = strings.ReplaceAll(in, `\\t`, "")
+ in = strings.ReplaceAll(in, `\n`, "\n")
+ in = strings.ReplaceAll(in, `\\r\\n`, "\n")
+ in = strings.ReplaceAll(in, `\r\n`, "\n")
+ in = strings.ReplaceAll(in, "\r\n", "\n")
+ in = strings.ReplaceAll(in, `\\r`, "\n")
+ in = strings.ReplaceAll(in, "\r", "\n")
+ in = strings.ReplaceAll(in, `\r`, "\n")
+ in = strings.ReplaceAll(in, `\\n`, "\n")
+ in = strings.ReplaceAll(in, `\n\n`, "\n")
+ in = strings.ReplaceAll(in, "\n\n", "\n")
+ in = strings.ReplaceAll(in, `\\`, "\n")
+
+ cleaned := strings.Builder{}
+ parts := strings.Split(in, "\n")
+ for _, line := range parts {
+ cleaned.WriteString(strings.TrimSpace(line) + "\n")
+ }
+ return cleaned.String()
+}
diff --git a/pkg/detectors/privatekey/privatekey.go b/pkg/detectors/privatekey/privatekey.go
new file mode 100644
index 000000000..10e88af90
--- /dev/null
+++ b/pkg/detectors/privatekey/privatekey.go
@@ -0,0 +1,146 @@
+package privatekey
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+ "time"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct {
+ IncludeExpired bool
+}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+ keyPat = regexp.MustCompile(`(?i)-----\s*?BEGIN[ A-Z0-9_-]*?PRIVATE KEY\s*?-----[\s\S]*?----\s*?END[ A-Z0-9_-]*? PRIVATE KEY\s*?-----`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"private key"}
+}
+
+// FromData will find and optionally verify Privatekey secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
+ results := []detectors.Result{}
+ dataStr := string(data)
+
+ matches := keyPat.FindAllString(dataStr, -1)
+
+ for _, match := range matches {
+
+ token := normalize(match)
+
+ if len(token) < 64 {
+ continue
+ }
+
+ secret := detectors.Result{
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Raw: []byte(token),
+ Redacted: token[0:64],
+ }
+
+ fingerprint, err := FingerprintPEMKey([]byte(token))
+ if err != nil {
+ continue
+ }
+
+ if verify {
+ data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
+ if err != nil {
+ log.Warn(err)
+ return nil, err
+ }
+ secret.StructuredData = data
+ if data != nil {
+ secret.Verified = true
+ }
+ }
+
+ results = append(results, secret)
+ }
+
+ return results, nil
+}
+
+func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
+ req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
+ if err != nil {
+ return
+ }
+ res, err := client.Do(req)
+ if err != nil {
+ return
+ }
+ defer res.Body.Close()
+
+ results := DriftwoodResult{}
+ err = json.NewDecoder(res.Body).Decode(&results)
+ if err != nil {
+ return
+ }
+
+ seen := map[string]struct{}{}
+ for _, r := range results.CertificateResults {
+ if _, ok := seen[r.CertificateFingerprint]; ok {
+ continue
+ }
+ if !includeExpired && time.Since(r.ExpirationTimestamp) > 0 {
+ continue
+ }
+ if data == nil {
+ data = &detectorspb.StructuredData{}
+ }
+ if data.TlsPrivateKey == nil {
+ data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
+ }
+ data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
+ CertificateFingerprint: r.CertificateFingerprint,
+ ExpirationTimestamp: r.ExpirationTimestamp.Unix(),
+ VerificationUrl: fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
+ })
+ seen[r.CertificateFingerprint] = struct{}{}
+ }
+
+ for _, r := range results.GitHubSSHResults {
+ if _, ok := seen[r.Username]; ok {
+ continue
+ }
+ if data == nil {
+ data = &detectorspb.StructuredData{}
+ }
+ if data.GithubSshKey == nil {
+ data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
+ }
+ data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
+ User: r.Username,
+ })
+ seen[r.Username] = struct{}{}
+ }
+
+ return
+}
+
+type DriftwoodResult struct {
+ CertificateResults []struct {
+ CertificateFingerprint string `json:"CertificateFingerprint"`
+ ExpirationTimestamp time.Time `json:"ExpirationTimestamp"`
+ } `json:"CertificateResults"`
+ GitHubSSHResults []struct {
+ Username string `json:"Username"`
+ } `json:"GitHubSSHResults"`
+}
diff --git a/pkg/detectors/privatekey/privatekey_test.go b/pkg/detectors/privatekey/privatekey_test.go
new file mode 100644
index 000000000..f0955f652
--- /dev/null
+++ b/pkg/detectors/privatekey/privatekey_test.go
@@ -0,0 +1,263 @@
+package privatekey
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestPrivatekey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secretTLS := testSecrets.MustGetField(("PRIVATEKEY_TLS"))
+ secretGitHub := testSecrets.MustGetField(("PRIVATEKEY_GITHUB"))
+ secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: false,
+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",
+ },
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: false,
+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found TLS private key, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(secretTLS),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: true,
+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
+ StructuredData: &detectorspb.StructuredData{
+ TlsPrivateKey: []*detectorspb.TlsPrivateKey{
+ {
+ CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
+ VerificationUrl: "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
+ ExpirationTimestamp: 1497337731,
+ },
+ {
+ CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+ VerificationUrl: "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+ ExpirationTimestamp: 1528192832,
+ },
+ },
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found GitHub SSH private key, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(secretGitHub),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: true,
+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
+ StructuredData: &detectorspb.StructuredData{
+ GithubSshKey: []*detectorspb.GitHubSSHKey{
+ {
+ User: "thisisforgithub0",
+ },
+ },
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found encrypted GitHub SSH private key, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
+ b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
+ xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
+ utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
+ inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
+ OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
+ sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
+ NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
+ WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
+ lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
+ UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
+ uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
+ Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
+ SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
+ kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
+ wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
+ TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
+ nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
+ TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
+ 8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
+ wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
+ akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
+ ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
+ Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
+ e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
+ s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
+ Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
+ kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
+ Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
+ fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
+ S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
+ sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
+ c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
+ /90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
+ muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
+ iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
+ f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
+ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
+ -----END OPENSSH PRIVATE KEY-----
+ `),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: true,
+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
+ StructuredData: &detectorspb.StructuredData{
+ GithubSshKey: []*detectorspb.GitHubSSHKey{
+ {
+ User: "thisisforgithub0",
+ },
+ },
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{IncludeExpired: true}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
+
+func Test_lookupFingerprint(t *testing.T) {
+ tests := []struct {
+ name string
+ publicKeyFingerprintInHex string
+ wantFingerprints bool
+ wantErr bool
+ includeExpired bool
+ }{
+ {
+ name: "got some",
+ publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",
+ wantFingerprints: true,
+ wantErr: false,
+ includeExpired: true,
+ },
+ {
+ name: "got some",
+ publicKeyFingerprintInHex: "none",
+ wantFingerprints: false,
+ wantErr: false,
+ includeExpired: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotFingerprints, err := lookupFingerprint(tt.publicKeyFingerprintInHex, tt.includeExpired)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {
+ t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/privatekey/rockyou-15.txt b/pkg/detectors/privatekey/rockyou-15.txt
new file mode 100644
index 000000000..31140491c
--- /dev/null
+++ b/pkg/detectors/privatekey/rockyou-15.txt
@@ -0,0 +1,249 @@
+123456
+12345
+123456789
+password
+iloveyou
+princess
+1234567
+12345678
+abc123
+nicole
+daniel
+babygirl
+monkey
+lovely
+jessica
+654321
+michael
+ashley
+qwerty
+111111
+iloveu
+000000
+michelle
+tigger
+sunshine
+chocolate
+password1
+soccer
+anthony
+friends
+butterfly
+purple
+angel
+jordan
+liverpool
+justin
+loveme
+fuckyou
+123123
+football
+secret
+andrea
+carlos
+jennifer
+joshua
+bubbles
+1234567890
+superman
+hannah
+amanda
+loveyou
+pretty
+basketball
+andrew
+angels
+tweety
+flower
+playboy
+hello
+elizabeth
+hottie
+tinkerbell
+charlie
+samantha
+barbie
+chelsea
+lovers
+teamo
+jasmine
+brandon
+666666
+shadow
+melissa
+eminem
+matthew
+robert
+danielle
+forever
+family
+jonathan
+987654321
+computer
+whatever
+dragon
+vanessa
+cookie
+naruto
+summer
+sweety
+spongebob
+joseph
+junior
+softball
+taylor
+yellow
+daniela
+lauren
+mickey
+princesa
+alexandra
+alexis
+jesus
+estrella
+miguel
+william
+thomas
+beautiful
+mylove
+angela
+poohbear
+patrick
+iloveme
+sakura
+adrian
+alexander
+destiny
+christian
+121212
+sayang
+america
+dancer
+monica
+richard
+112233
+princess1
+555555
+diamond
+carolina
+steven
+rangers
+louise
+orange
+789456
+999999
+shorty
+11111
+nathan
+snoopy
+gabriel
+hunter
+cherry
+killer
+sandra
+alejandro
+buster
+george
+brittany
+alejandra
+patricia
+rachel
+tequiero
+7777777
+cheese
+159753
+arsenal
+dolphin
+antonio
+heather
+david
+ginger
+stephanie
+peanut
+blink182
+sweetie
+222222
+beauty
+987654
+victoria
+honey
+00000
+fernando
+pokemon
+maggie
+corazon
+chicken
+pepper
+cristina
+rainbow
+kisses
+manuel
+myspace
+rebelde
+angel1
+ricardo
+babygurl
+heaven
+55555
+baseball
+martin
+greenday
+november
+alyssa
+madison
+mother
+123321
+123abc
+mahalkita
+batman
+september
+december
+morgan
+mariposa
+maria
+gabriela
+iloveyou2
+bailey
+jeremy
+pamela
+kimberly
+gemini
+shannon
+pictures
+asshole
+sophie
+jessie
+hellokitty
+claudia
+babygirl1
+angelica
+austin
+mahalko
+victor
+horses
+tiffany
+mariana
+eduardo
+andres
+courtney
+booboo
+kissme
+harley
+ronaldo
+iloveyou1
+precious
+october
+inuyasha
+peaches
+veronica
+chris
+888888
+adriana
+cutie
+james
+banana
+prince
+friend
+jesus1
+crystal
+celtic
\ No newline at end of file
diff --git a/pkg/detectors/prospectcrm/prospectcrm_test.go b/pkg/detectors/prospectcrm/prospectcrm_test.go
index fed2d0ead..205537d89 100644
--- a/pkg/detectors/prospectcrm/prospectcrm_test.go
+++ b/pkg/detectors/prospectcrm/prospectcrm_test.go
@@ -16,7 +16,7 @@ import (
func TestProspectCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/prospectio/prospectio_test.go b/pkg/detectors/prospectio/prospectio_test.go
index cb38ebacb..181ed7ae8 100644
--- a/pkg/detectors/prospectio/prospectio_test.go
+++ b/pkg/detectors/prospectio/prospectio_test.go
@@ -16,7 +16,7 @@ import (
func TestProspectIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/protocolsio/protocolsio.go b/pkg/detectors/protocolsio/protocolsio.go
new file mode 100644
index 000000000..e5315f7b2
--- /dev/null
+++ b/pkg/detectors/protocolsio/protocolsio.go
@@ -0,0 +1,74 @@
+package protocolsio
+
+import (
+ "context"
+ "fmt"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"protocols"}) + `\b([a-z0-9]{64})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"protocols"}
+}
+
+// FromData will find and optionally verify ProtocolsIO secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_ProtocolsIO,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://www.protocols.io/api/v3/session/profile", nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/protocolsio/protocolsio_test.go b/pkg/detectors/protocolsio/protocolsio_test.go
new file mode 100644
index 000000000..0cabbfce4
--- /dev/null
+++ b/pkg/detectors/protocolsio/protocolsio_test.go
@@ -0,0 +1,113 @@
+package protocolsio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestProtocolsIO_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PROTOCOLSIO")
+ inactiveSecret := testSecrets.MustGetField("PROTOCOLSIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProtocolsIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProtocolsIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ProtocolsIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ProtocolsIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/proxycrawl/proxycrawl_test.go b/pkg/detectors/proxycrawl/proxycrawl_test.go
index 394976f3c..8ecea2bef 100644
--- a/pkg/detectors/proxycrawl/proxycrawl_test.go
+++ b/pkg/detectors/proxycrawl/proxycrawl_test.go
@@ -16,7 +16,7 @@ import (
func TestProxyCrawl_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go
index d4a6c8a45..e2fbdc45d 100644
--- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go
+++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go
@@ -16,7 +16,7 @@ import (
func TestPubNubPublishKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go
index 820842f62..4f8bbfa43 100644
--- a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go
+++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go
@@ -16,7 +16,7 @@ import (
func TestPubNubSubscriptionKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/purestake/purestake_test.go b/pkg/detectors/purestake/purestake_test.go
index 47eab6ad9..d28fde2fc 100644
--- a/pkg/detectors/purestake/purestake_test.go
+++ b/pkg/detectors/purestake/purestake_test.go
@@ -16,7 +16,7 @@ import (
func TestPureStake_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go
index f1d32a0f7..d786a495f 100644
--- a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go
+++ b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestPushBulletApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go
index 9fc558076..8752e5e12 100644
--- a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go
+++ b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go
@@ -16,7 +16,7 @@ import (
func TestPusherChannelKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/qualaroo/qualaroo_test.go b/pkg/detectors/qualaroo/qualaroo_test.go
index 020f80c19..1bb856092 100644
--- a/pkg/detectors/qualaroo/qualaroo_test.go
+++ b/pkg/detectors/qualaroo/qualaroo_test.go
@@ -16,7 +16,7 @@ import (
func TestQualaroo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/qubole/qubole_test.go b/pkg/detectors/qubole/qubole_test.go
index cd015dc8c..bad932f44 100644
--- a/pkg/detectors/qubole/qubole_test.go
+++ b/pkg/detectors/qubole/qubole_test.go
@@ -16,7 +16,7 @@ import (
func TestQubole_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/quickmetrics/quickmetrics_test.go b/pkg/detectors/quickmetrics/quickmetrics_test.go
index a6acfcd10..d40cdb613 100644
--- a/pkg/detectors/quickmetrics/quickmetrics_test.go
+++ b/pkg/detectors/quickmetrics/quickmetrics_test.go
@@ -16,7 +16,7 @@ import (
func TestQuickMetrics_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/rapidapi/rapidapi_test.go b/pkg/detectors/rapidapi/rapidapi_test.go
index 0724f1f5e..d64586337 100644
--- a/pkg/detectors/rapidapi/rapidapi_test.go
+++ b/pkg/detectors/rapidapi/rapidapi_test.go
@@ -16,7 +16,7 @@ import (
func TestRapidApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/raven/raven_test.go b/pkg/detectors/raven/raven_test.go
index e214c6996..c83caaa5c 100644
--- a/pkg/detectors/raven/raven_test.go
+++ b/pkg/detectors/raven/raven_test.go
@@ -16,7 +16,7 @@ import (
func TestRaven_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/rawg/rawg_test.go b/pkg/detectors/rawg/rawg_test.go
index 9263627f2..90eb0ad44 100644
--- a/pkg/detectors/rawg/rawg_test.go
+++ b/pkg/detectors/rawg/rawg_test.go
@@ -16,7 +16,7 @@ import (
func TestRawg_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.go
new file mode 100644
index 000000000..2df80bf9d
--- /dev/null
+++ b/pkg/detectors/razorpay/razorpay.go
@@ -0,0 +1,86 @@
+package razorpay
+
+import (
+ "context"
+ "regexp"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ razorpay "github.com/razorpay/razorpay-go"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+//The (`) character adds secondary encoding to parsed strings by Golang which also allows for escape sequences
+var (
+ keyPat = regexp.MustCompile(`(?i)\brzp_\w{2,6}_\w{10,20}\b`)
+ secretPat = regexp.MustCompile(`(?:razor|secret|rzp|key)[-\w]*[\" :=']*([A-Za-z0-9]{20,50})`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"rzp_"}
+}
+
+// FromData will find and optionally verify RazorPay secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllString(dataStr, -1)
+
+ for _, match := range matches {
+ token := match
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_RazorPay,
+ Raw: []byte(token),
+ Redacted: token,
+ }
+
+ if verify {
+ //https://dashboard.razorpay.com/#/access/signin
+ //https://gitlab.com/trufflesec/trufflehog/-/blob/master/webapi/secrets/razorpay.py
+ secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+ if len(secMatches) == 0 {
+ //no secret keys were found. Declare unverified (This is how AWS secret handles the same logic)
+ //TODO determine if key alone without secret is reportable
+ s.Verified = false
+ return
+ }
+ //we only want the secret, not its surrounding info - grabbing capture groups
+ for _, secMatch := range secMatches {
+ client := razorpay.NewClient(token, secMatch[1])
+ resp, err := client.Order.All(nil, nil)
+ //TODO Error handling is broken in SDK, fixed by https://github.com/razorpay/razorpay-go/pull/23
+ //waiting to be reviewed and merged
+ if resp == nil {
+ continue
+ }
+ if err != nil {
+ log.Debugf("Error verifying likely razorpay key/secret combo: %v", err)
+ continue
+ }
+ //TODO debug with responses. could still be invalid at this stage
+
+ s.Verified = true
+ // fmt.Println(resp)
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/razorpay/razorpay_test.go b/pkg/detectors/razorpay/razorpay_test.go
new file mode 100644
index 000000000..67c41d37b
--- /dev/null
+++ b/pkg/detectors/razorpay/razorpay_test.go
@@ -0,0 +1,100 @@
+package razorpay
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestRazorPay_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secretInactive := testSecrets.MustGetField("RAZORPAY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_test_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RazorPay,
+ Redacted: "rzp_test_SnDTaP1ncfliDt",
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RazorPay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/readme/readme_test.go b/pkg/detectors/readme/readme_test.go
index cf09fe71a..124938c13 100644
--- a/pkg/detectors/readme/readme_test.go
+++ b/pkg/detectors/readme/readme_test.go
@@ -16,7 +16,7 @@ import (
func TestReadMe_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go
index f89322ed6..54652caa9 100644
--- a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go
+++ b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go
@@ -16,7 +16,7 @@ import (
func TestReallySimpleSystems_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/rebrandly/rebrandly_test.go b/pkg/detectors/rebrandly/rebrandly_test.go
index 7d27ee6a9..f76c6bde4 100644
--- a/pkg/detectors/rebrandly/rebrandly_test.go
+++ b/pkg/detectors/rebrandly/rebrandly_test.go
@@ -16,7 +16,7 @@ import (
func TestRebrandly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/refiner/refiner_test.go b/pkg/detectors/refiner/refiner_test.go
index f7d19faec..202352caa 100644
--- a/pkg/detectors/refiner/refiner_test.go
+++ b/pkg/detectors/refiner/refiner_test.go
@@ -16,7 +16,7 @@ import (
func TestRefiner_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/restpack/restpack_test.go b/pkg/detectors/restpack/restpack_test.go
index b6a7621db..c0d07c0cc 100644
--- a/pkg/detectors/restpack/restpack_test.go
+++ b/pkg/detectors/restpack/restpack_test.go
@@ -16,7 +16,7 @@ import (
func TestRestpack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go
index 3187ec0dd..b99fe92ac 100644
--- a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go
+++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go
@@ -16,7 +16,7 @@ import (
func TestRestpackHtmlToPdfAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go
index a3a8a94a1..00beece9e 100644
--- a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go
+++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go
@@ -16,7 +16,7 @@ import (
func TestRestpackScreenshotAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/rev/rev_test.go b/pkg/detectors/rev/rev_test.go
index 5e5dd23f9..b83dd4e57 100644
--- a/pkg/detectors/rev/rev_test.go
+++ b/pkg/detectors/rev/rev_test.go
@@ -16,7 +16,7 @@ import (
func TestRev_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/revampcrm/revampcrm_test.go b/pkg/detectors/revampcrm/revampcrm_test.go
index 807cbba6b..9cbc4c5a8 100644
--- a/pkg/detectors/revampcrm/revampcrm_test.go
+++ b/pkg/detectors/revampcrm/revampcrm_test.go
@@ -16,7 +16,7 @@ import (
func TestRevampCRM_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ringcentral/ringcentral_test.go b/pkg/detectors/ringcentral/ringcentral_test.go
index 32205f0da..23a3b6d28 100644
--- a/pkg/detectors/ringcentral/ringcentral_test.go
+++ b/pkg/detectors/ringcentral/ringcentral_test.go
@@ -16,7 +16,7 @@ import (
func TestRingcentral_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ritekit/ritekit_test.go b/pkg/detectors/ritekit/ritekit_test.go
index fde353550..e17c253d5 100644
--- a/pkg/detectors/ritekit/ritekit_test.go
+++ b/pkg/detectors/ritekit/ritekit_test.go
@@ -16,7 +16,7 @@ import (
func TestRiteKit_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/roaring/roaring_test.go b/pkg/detectors/roaring/roaring_test.go
index 17ee0f19a..2323a5b23 100644
--- a/pkg/detectors/roaring/roaring_test.go
+++ b/pkg/detectors/roaring/roaring_test.go
@@ -16,7 +16,7 @@ import (
func TestRoaring_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/rocketreach/rocketreach_test.go b/pkg/detectors/rocketreach/rocketreach_test.go
index 7da7c8f5f..a34b7ccaa 100644
--- a/pkg/detectors/rocketreach/rocketreach_test.go
+++ b/pkg/detectors/rocketreach/rocketreach_test.go
@@ -16,7 +16,7 @@ import (
func TestRocketReach_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/roninapp/roninapp_test.go b/pkg/detectors/roninapp/roninapp_test.go
index 6fbd22995..9e99594e3 100644
--- a/pkg/detectors/roninapp/roninapp_test.go
+++ b/pkg/detectors/roninapp/roninapp_test.go
@@ -16,7 +16,7 @@ import (
func TestRoninApp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/route4me/route4me_test.go b/pkg/detectors/route4me/route4me_test.go
index 8b98a68e4..2355b393c 100644
--- a/pkg/detectors/route4me/route4me_test.go
+++ b/pkg/detectors/route4me/route4me_test.go
@@ -16,7 +16,7 @@ import (
func TestRoute4me_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/rubygems/rubygems_test.go b/pkg/detectors/rubygems/rubygems_test.go
index ebd564af5..b8153f2c3 100644
--- a/pkg/detectors/rubygems/rubygems_test.go
+++ b/pkg/detectors/rubygems/rubygems_test.go
@@ -16,7 +16,7 @@ import (
func TestRubyGems_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/runrunit/runrunit_test.go b/pkg/detectors/runrunit/runrunit_test.go
index ae25716d1..f554ad35c 100644
--- a/pkg/detectors/runrunit/runrunit_test.go
+++ b/pkg/detectors/runrunit/runrunit_test.go
@@ -16,7 +16,7 @@ import (
func TestRunRunIt_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/salesblink/salesblink_test.go b/pkg/detectors/salesblink/salesblink_test.go
index 88a9a3127..b585007a6 100644
--- a/pkg/detectors/salesblink/salesblink_test.go
+++ b/pkg/detectors/salesblink/salesblink_test.go
@@ -16,7 +16,7 @@ import (
func TestSalesblink_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/salescookie/salescookie_test.go b/pkg/detectors/salescookie/salescookie_test.go
index e05c4bc6e..78749935a 100644
--- a/pkg/detectors/salescookie/salescookie_test.go
+++ b/pkg/detectors/salescookie/salescookie_test.go
@@ -16,7 +16,7 @@ import (
func TestSalescookie_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/salesflare/salesflare_test.go b/pkg/detectors/salesflare/salesflare_test.go
index 98a95a11b..7604af877 100644
--- a/pkg/detectors/salesflare/salesflare_test.go
+++ b/pkg/detectors/salesflare/salesflare_test.go
@@ -16,7 +16,7 @@ import (
func TestSalesflare_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go b/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go
index be7c60e8c..e6f02e603 100644
--- a/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go
+++ b/pkg/detectors/satismeterprojectkey/satismeterprojectkey_test.go
@@ -16,7 +16,7 @@ import (
func TestSatismeterProjectkey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go b/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go
index 957ae1c0b..394861ca7 100644
--- a/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go
+++ b/pkg/detectors/satismeterwritekey/satismeterwritekey_test.go
@@ -16,7 +16,7 @@ import (
func TestSatismeterWritekey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/saucelabs/saucelabs_test.go b/pkg/detectors/saucelabs/saucelabs_test.go
index d394cd000..2445ce46a 100644
--- a/pkg/detectors/saucelabs/saucelabs_test.go
+++ b/pkg/detectors/saucelabs/saucelabs_test.go
@@ -16,7 +16,7 @@ import (
func TestSauceLabs_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scalewaykey/scalewaykey_test.go b/pkg/detectors/scalewaykey/scalewaykey_test.go
index 4aff92aa7..573757f0c 100644
--- a/pkg/detectors/scalewaykey/scalewaykey_test.go
+++ b/pkg/detectors/scalewaykey/scalewaykey_test.go
@@ -16,7 +16,7 @@ import (
func TestScalewayKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scrapeowl/scrapeowl_test.go b/pkg/detectors/scrapeowl/scrapeowl_test.go
index ce0e77f50..bc31f6103 100644
--- a/pkg/detectors/scrapeowl/scrapeowl_test.go
+++ b/pkg/detectors/scrapeowl/scrapeowl_test.go
@@ -16,7 +16,7 @@ import (
func TestScrapeowl_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scraperapi/scraperapi_test.go b/pkg/detectors/scraperapi/scraperapi_test.go
index dd8368050..386240e10 100644
--- a/pkg/detectors/scraperapi/scraperapi_test.go
+++ b/pkg/detectors/scraperapi/scraperapi_test.go
@@ -16,7 +16,7 @@ import (
func TestScraperAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scraperbox/scraperbox_test.go b/pkg/detectors/scraperbox/scraperbox_test.go
index a673ccb2f..1f96fb5a7 100644
--- a/pkg/detectors/scraperbox/scraperbox_test.go
+++ b/pkg/detectors/scraperbox/scraperbox_test.go
@@ -16,7 +16,7 @@ import (
func TestScraperBox_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scrapersite/scrapersite_test.go b/pkg/detectors/scrapersite/scrapersite_test.go
index f92a967c5..f882ab3d0 100644
--- a/pkg/detectors/scrapersite/scrapersite_test.go
+++ b/pkg/detectors/scrapersite/scrapersite_test.go
@@ -16,7 +16,7 @@ import (
func TestScraperSite_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scrapestack/scrapestack_test.go b/pkg/detectors/scrapestack/scrapestack_test.go
index 49f2867ee..b81893c01 100644
--- a/pkg/detectors/scrapestack/scrapestack_test.go
+++ b/pkg/detectors/scrapestack/scrapestack_test.go
@@ -16,7 +16,7 @@ import (
func TestScrapeStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scrapfly/scrapfly_test.go b/pkg/detectors/scrapfly/scrapfly_test.go
index bc0396e61..ff8e402c5 100644
--- a/pkg/detectors/scrapfly/scrapfly_test.go
+++ b/pkg/detectors/scrapfly/scrapfly_test.go
@@ -16,7 +16,7 @@ import (
func TestScrapfly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scrapingant/scrapingant_test.go b/pkg/detectors/scrapingant/scrapingant_test.go
index 41b37fb8d..7c228d924 100644
--- a/pkg/detectors/scrapingant/scrapingant_test.go
+++ b/pkg/detectors/scrapingant/scrapingant_test.go
@@ -16,7 +16,7 @@ import (
func TestScrapingAnt_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/scrapingbee/scrapingbee_test.go b/pkg/detectors/scrapingbee/scrapingbee_test.go
index c22b82dd6..7b257c573 100644
--- a/pkg/detectors/scrapingbee/scrapingbee_test.go
+++ b/pkg/detectors/scrapingbee/scrapingbee_test.go
@@ -16,7 +16,7 @@ import (
func TestScrapingBee_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/screenshotapi/screenshotapi_test.go b/pkg/detectors/screenshotapi/screenshotapi_test.go
index 0a2ad4015..9eb2861d0 100644
--- a/pkg/detectors/screenshotapi/screenshotapi_test.go
+++ b/pkg/detectors/screenshotapi/screenshotapi_test.go
@@ -16,7 +16,7 @@ import (
func TestScreenshotAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/screenshotlayer/screenshotlayer_test.go b/pkg/detectors/screenshotlayer/screenshotlayer_test.go
index 55f0e5452..0ae9c721a 100644
--- a/pkg/detectors/screenshotlayer/screenshotlayer_test.go
+++ b/pkg/detectors/screenshotlayer/screenshotlayer_test.go
@@ -16,7 +16,7 @@ import (
func TestScreenshotLayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/securitytrails/securitytrails_test.go b/pkg/detectors/securitytrails/securitytrails_test.go
index a62219894..4cb4efcb6 100644
--- a/pkg/detectors/securitytrails/securitytrails_test.go
+++ b/pkg/detectors/securitytrails/securitytrails_test.go
@@ -16,7 +16,7 @@ import (
func TestSecurityTrails_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/segmentapikey/segmentapikey_test.go b/pkg/detectors/segmentapikey/segmentapikey_test.go
index 23901eb59..fd8c03dc3 100644
--- a/pkg/detectors/segmentapikey/segmentapikey_test.go
+++ b/pkg/detectors/segmentapikey/segmentapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestSegmentApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/selectpdf/selectpdf_test.go b/pkg/detectors/selectpdf/selectpdf_test.go
index 082d38941..c1600228d 100644
--- a/pkg/detectors/selectpdf/selectpdf_test.go
+++ b/pkg/detectors/selectpdf/selectpdf_test.go
@@ -16,7 +16,7 @@ import (
func TestSelectPDF_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/semaphore/semaphore_test.go b/pkg/detectors/semaphore/semaphore_test.go
index 6a113e498..8c2915d26 100644
--- a/pkg/detectors/semaphore/semaphore_test.go
+++ b/pkg/detectors/semaphore/semaphore_test.go
@@ -16,7 +16,7 @@ import (
func TestSemaphore_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sendgrid/sendgrid.go b/pkg/detectors/sendgrid/sendgrid.go
new file mode 100644
index 000000000..5c112b1bb
--- /dev/null
+++ b/pkg/detectors/sendgrid/sendgrid.go
@@ -0,0 +1,83 @@
+package sendgrid
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.RetryableHttpClient()
+
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"SG."}
+}
+
+// FromData will find and optionally verify Sendgrid secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ res := strings.TrimSpace(match[1])
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Raw: []byte(res),
+ }
+
+ if verify {
+ // there are a few endpoints we can check, but templates seems the least sensitive.
+ // 403 will be issued if the scope is wrong but the key is correct
+ baseURL := "https://api.sendgrid.com/v3/templates"
+
+ // test `read_user` scope
+ req, _ := http.NewRequest("GET", baseURL, nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", res))
+ req.Header.Add("Content-Type", "application/json")
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ // 200 means good key and has `templates` scope
+ // 403 means good key but not the right scope
+ // 401 is bad key
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/sendgrid/sendgrid_test.go b/pkg/detectors/sendgrid/sendgrid_test.go
new file mode 100644
index 000000000..e567a746d
--- /dev/null
+++ b/pkg/detectors/sendgrid/sendgrid_test.go
@@ -0,0 +1,108 @@
+package sendgrid
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestSendgrid_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SENDGRID")
+ secretInactive := testSecrets.MustGetField("SENDGRID_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sendinbluev2/sendinbluev2_test.go b/pkg/detectors/sendinbluev2/sendinbluev2_test.go
index 0645a7e22..36e1e17a7 100644
--- a/pkg/detectors/sendinbluev2/sendinbluev2_test.go
+++ b/pkg/detectors/sendinbluev2/sendinbluev2_test.go
@@ -16,7 +16,7 @@ import (
func TestSendinBlueV2_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sentiment/sentiment_test.go b/pkg/detectors/sentiment/sentiment_test.go
index c616371c9..6f0ea0c46 100644
--- a/pkg/detectors/sentiment/sentiment_test.go
+++ b/pkg/detectors/sentiment/sentiment_test.go
@@ -16,7 +16,7 @@ import (
func TestSentiment_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sentrytoken/sentrytoken_test.go b/pkg/detectors/sentrytoken/sentrytoken_test.go
index 1c2fd74f6..27c1122b7 100644
--- a/pkg/detectors/sentrytoken/sentrytoken_test.go
+++ b/pkg/detectors/sentrytoken/sentrytoken_test.go
@@ -16,7 +16,7 @@ import (
func TestSentryToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/serphouse/serphouse_test.go b/pkg/detectors/serphouse/serphouse_test.go
index 52fa32959..fa90bbb02 100644
--- a/pkg/detectors/serphouse/serphouse_test.go
+++ b/pkg/detectors/serphouse/serphouse_test.go
@@ -16,7 +16,7 @@ import (
func TestSerphouse_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/serpstack/serpstack_test.go b/pkg/detectors/serpstack/serpstack_test.go
index 287779dd4..a12c91756 100644
--- a/pkg/detectors/serpstack/serpstack_test.go
+++ b/pkg/detectors/serpstack/serpstack_test.go
@@ -16,7 +16,7 @@ import (
func TestSerpStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sheety/sheety_test.go b/pkg/detectors/sheety/sheety_test.go
index 2b2419546..5316e2cfc 100644
--- a/pkg/detectors/sheety/sheety_test.go
+++ b/pkg/detectors/sheety/sheety_test.go
@@ -16,7 +16,7 @@ import (
func TestSheety_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sherpadesk/sherpadesk_test.go b/pkg/detectors/sherpadesk/sherpadesk_test.go
index 31e7d0f53..295c7edcc 100644
--- a/pkg/detectors/sherpadesk/sherpadesk_test.go
+++ b/pkg/detectors/sherpadesk/sherpadesk_test.go
@@ -16,7 +16,7 @@ import (
func TestSherpadesk_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/shipday/shipday_test.go b/pkg/detectors/shipday/shipday_test.go
index 342df1d84..ef0a9b634 100644
--- a/pkg/detectors/shipday/shipday_test.go
+++ b/pkg/detectors/shipday/shipday_test.go
@@ -16,7 +16,7 @@ import (
func TestShipday_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/shodankey/shodankey_test.go b/pkg/detectors/shodankey/shodankey_test.go
index 750759e6c..2b995ff28 100644
--- a/pkg/detectors/shodankey/shodankey_test.go
+++ b/pkg/detectors/shodankey/shodankey_test.go
@@ -16,7 +16,7 @@ import (
func TestShodanKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/shortcut/shortcut_test.go b/pkg/detectors/shortcut/shortcut_test.go
index 2b159b6d0..37a2df726 100644
--- a/pkg/detectors/shortcut/shortcut_test.go
+++ b/pkg/detectors/shortcut/shortcut_test.go
@@ -16,7 +16,7 @@ import (
func TestShortcut_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/shotstack/shotstack_test.go b/pkg/detectors/shotstack/shotstack_test.go
index a798da854..351f0e733 100644
--- a/pkg/detectors/shotstack/shotstack_test.go
+++ b/pkg/detectors/shotstack/shotstack_test.go
@@ -16,7 +16,7 @@ import (
func TestShotstack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/shutterstock/shutterstock_test.go b/pkg/detectors/shutterstock/shutterstock_test.go
index 0ddb49b6a..6353f8c30 100644
--- a/pkg/detectors/shutterstock/shutterstock_test.go
+++ b/pkg/detectors/shutterstock/shutterstock_test.go
@@ -16,7 +16,7 @@ import (
func TestShutterstock_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go
index 896b269c1..c5d07da93 100644
--- a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go
+++ b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go
@@ -16,7 +16,7 @@ import (
func TestShutterstockOAuth_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.go
index e06dd792b..d82ed4a4b 100644
--- a/pkg/detectors/signalwire/signalwire_test.go
+++ b/pkg/detectors/signalwire/signalwire_test.go
@@ -16,7 +16,7 @@ import (
func TestSignalwire_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/signaturit/signaturit_test.go b/pkg/detectors/signaturit/signaturit_test.go
index 187ee3745..1c2a01714 100644
--- a/pkg/detectors/signaturit/signaturit_test.go
+++ b/pkg/detectors/signaturit/signaturit_test.go
@@ -16,7 +16,7 @@ import (
func TestSignaturit_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/signupgenius/signupgenius_test.go b/pkg/detectors/signupgenius/signupgenius_test.go
index 669bc059b..edfaf0bd9 100644
--- a/pkg/detectors/signupgenius/signupgenius_test.go
+++ b/pkg/detectors/signupgenius/signupgenius_test.go
@@ -16,7 +16,7 @@ import (
func TestSignupgenius_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sigopt/sigopt.go b/pkg/detectors/sigopt/sigopt.go
new file mode 100644
index 000000000..76315fc83
--- /dev/null
+++ b/pkg/detectors/sigopt/sigopt.go
@@ -0,0 +1,78 @@
+package sigopt
+
+import (
+ "context"
+ b64 "encoding/base64"
+ "fmt"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sigopt"}) + `\b([A-Z0-9]{48})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"sigopt"}
+}
+
+// FromData will find and optionally verify Sigopt secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Sigopt,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ data := fmt.Sprintf("%s:", resMatch)
+ sEnc := b64.StdEncoding.EncodeToString([]byte(data))
+ req, _ := http.NewRequest("GET", "https://api.sigopt.com/v1/experiments", nil)
+ req.Header.Add("Content-Type", "application/json")
+ req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/sigopt/sigopt_test.go b/pkg/detectors/sigopt/sigopt_test.go
new file mode 100644
index 000000000..9215595b2
--- /dev/null
+++ b/pkg/detectors/sigopt/sigopt_test.go
@@ -0,0 +1,113 @@
+package sigopt
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestSigopt_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIGOPT")
+ inactiveSecret := testSecrets.MustGetField("SIGOPT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sigopt,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sigopt,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sigopt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sigopt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/simplesat/simplesat_test.go b/pkg/detectors/simplesat/simplesat_test.go
index 417810263..2857f3fa3 100644
--- a/pkg/detectors/simplesat/simplesat_test.go
+++ b/pkg/detectors/simplesat/simplesat_test.go
@@ -16,7 +16,7 @@ import (
func TestSimplesat_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/simplynoted/simplynoted_test.go b/pkg/detectors/simplynoted/simplynoted_test.go
index b0414d1d9..247b3d955 100644
--- a/pkg/detectors/simplynoted/simplynoted_test.go
+++ b/pkg/detectors/simplynoted/simplynoted_test.go
@@ -16,7 +16,7 @@ import (
func TestSimplyNoted_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/simvoly/simvoly_test.go b/pkg/detectors/simvoly/simvoly_test.go
index d6a9bd8ab..60be5d1a0 100644
--- a/pkg/detectors/simvoly/simvoly_test.go
+++ b/pkg/detectors/simvoly/simvoly_test.go
@@ -16,7 +16,7 @@ import (
func TestSimvoly_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sinchmessage/sinchmessage_test.go b/pkg/detectors/sinchmessage/sinchmessage_test.go
index 49a667275..ed428f829 100644
--- a/pkg/detectors/sinchmessage/sinchmessage_test.go
+++ b/pkg/detectors/sinchmessage/sinchmessage_test.go
@@ -16,7 +16,7 @@ import (
func TestSinchMessage_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/siteleaf/siteleaf_test.go b/pkg/detectors/siteleaf/siteleaf_test.go
index 3dc42f9be..32f027789 100644
--- a/pkg/detectors/siteleaf/siteleaf_test.go
+++ b/pkg/detectors/siteleaf/siteleaf_test.go
@@ -16,7 +16,7 @@ import (
func TestSiteleaf_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/skrappio/skrappio_test.go b/pkg/detectors/skrappio/skrappio_test.go
index aea509d00..410e494d1 100644
--- a/pkg/detectors/skrappio/skrappio_test.go
+++ b/pkg/detectors/skrappio/skrappio_test.go
@@ -16,7 +16,7 @@ import (
func TestSkrapio_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/skybiometry/skybiometry_test.go b/pkg/detectors/skybiometry/skybiometry_test.go
index da193275a..50973fa56 100644
--- a/pkg/detectors/skybiometry/skybiometry_test.go
+++ b/pkg/detectors/skybiometry/skybiometry_test.go
@@ -16,7 +16,7 @@ import (
func TestSkyBiometry_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/slack/slack.go b/pkg/detectors/slack/slack.go
new file mode 100644
index 000000000..4556ffb97
--- /dev/null
+++ b/pkg/detectors/slack/slack.go
@@ -0,0 +1,79 @@
+package slack
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Check that the Slack scanner implements the SecretScanner interface at compile time.
+var _ detectors.Detector = Scanner{}
+
+var (
+ tokenPats = map[string]*regexp.Regexp{
+ "Slack Bot Token": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),
+ "Slack User Token": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),
+ "Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),
+ "Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-z0-9\-]*`),
+ }
+ verifyURL = "https://slack.com/api/auth.test"
+)
+
+type authRes struct {
+ Ok bool `json:"ok"`
+ URL string `json:"url"` // Workspace URL
+ Team string `json:"team"` // Human friendly workspace name
+ User string `json:"user"` // Username
+ TeamID string `json:"team_id"`
+ UserID string `json:"user_id"`
+ BotID string `json:"bot_id"`
+ Error string `json:"error"`
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"xoxb-", "xoxp-", "xoxa-", "xoxr-"}
+}
+
+// FromData will find and optionally verify Slack secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ for _, tokenPat := range tokenPats {
+ tokens := tokenPat.FindAllString(dataStr, -1)
+
+ for _, token := range tokens {
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Slack,
+ Raw: []byte(token),
+ }
+ if verify {
+ client := common.RetryableHttpClient()
+ req, _ := http.NewRequest("POST", verifyURL, nil)
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ var authResponse authRes
+ json.NewDecoder(res.Body).Decode(&authResponse)
+ s.Verified = authResponse.Ok
+ }
+ }
+
+ results = append(results, s)
+ }
+ }
+
+ return
+}
diff --git a/pkg/detectors/slack/slack_test.go b/pkg/detectors/slack/slack_test.go
new file mode 100644
index 000000000..b94f209c8
--- /dev/null
+++ b/pkg/detectors/slack/slack_test.go
@@ -0,0 +1,87 @@
+package slack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestScanner_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SLACK")
+ secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
+ tests := []struct {
+ name string
+ data []byte
+ verify bool
+ wantResults []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+ verify: true,
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found but unverified",
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
+ verify: true,
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(context.Background(), tt.verify, tt.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.wantResults); diff != "" {
+ t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/slackwebhook/slackwebhook_test.go b/pkg/detectors/slackwebhook/slackwebhook_test.go
index ce035f170..92e6ac6dc 100644
--- a/pkg/detectors/slackwebhook/slackwebhook_test.go
+++ b/pkg/detectors/slackwebhook/slackwebhook_test.go
@@ -16,7 +16,7 @@ import (
func TestSlackWebhook_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/smartsheets/smartsheets_test.go b/pkg/detectors/smartsheets/smartsheets_test.go
index bbf1f214f..07829f78c 100644
--- a/pkg/detectors/smartsheets/smartsheets_test.go
+++ b/pkg/detectors/smartsheets/smartsheets_test.go
@@ -16,7 +16,7 @@ import (
func TestSmartsheets_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/smartystreets/smartystreets_test.go b/pkg/detectors/smartystreets/smartystreets_test.go
index d5eaf08d0..cac5896f9 100644
--- a/pkg/detectors/smartystreets/smartystreets_test.go
+++ b/pkg/detectors/smartystreets/smartystreets_test.go
@@ -16,7 +16,7 @@ import (
func TestSmartyStreets_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/smooch/smooch_test.go b/pkg/detectors/smooch/smooch_test.go
index a74e35945..c1b0fd4bd 100644
--- a/pkg/detectors/smooch/smooch_test.go
+++ b/pkg/detectors/smooch/smooch_test.go
@@ -16,7 +16,7 @@ import (
func TestSmooch_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/snipcart/snipcart_test.go b/pkg/detectors/snipcart/snipcart_test.go
index cfbd86377..4e086c8ad 100644
--- a/pkg/detectors/snipcart/snipcart_test.go
+++ b/pkg/detectors/snipcart/snipcart_test.go
@@ -16,7 +16,7 @@ import (
func TestSnipcart_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/snykkey/snykkey_test.go b/pkg/detectors/snykkey/snykkey_test.go
index cbc1e2239..027da5fd2 100644
--- a/pkg/detectors/snykkey/snykkey_test.go
+++ b/pkg/detectors/snykkey/snykkey_test.go
@@ -16,7 +16,7 @@ import (
func TestSnykKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sparkpost/sparkpost_test.go b/pkg/detectors/sparkpost/sparkpost_test.go
index 8afaf237e..8a58d3f68 100644
--- a/pkg/detectors/sparkpost/sparkpost_test.go
+++ b/pkg/detectors/sparkpost/sparkpost_test.go
@@ -16,7 +16,7 @@ import (
func TestSparkpost_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go
index 5225d09f2..7393e7253 100644
--- a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go
+++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go
@@ -16,7 +16,7 @@ import (
func TestSplunkObservabilityToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/spoonacular/spoonacular_test.go b/pkg/detectors/spoonacular/spoonacular_test.go
index 667d1e897..561df5149 100644
--- a/pkg/detectors/spoonacular/spoonacular_test.go
+++ b/pkg/detectors/spoonacular/spoonacular_test.go
@@ -16,7 +16,7 @@ import (
func TestSpoonacular_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sportsmonk/sportsmonk_test.go b/pkg/detectors/sportsmonk/sportsmonk_test.go
index a91ba6935..e1e55da16 100644
--- a/pkg/detectors/sportsmonk/sportsmonk_test.go
+++ b/pkg/detectors/sportsmonk/sportsmonk_test.go
@@ -16,7 +16,7 @@ import (
func TestSportsmonk_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/spotifykey/spotifykey_test.go b/pkg/detectors/spotifykey/spotifykey_test.go
index dca79f10e..04316a378 100644
--- a/pkg/detectors/spotifykey/spotifykey_test.go
+++ b/pkg/detectors/spotifykey/spotifykey_test.go
@@ -16,7 +16,7 @@ import (
func TestSpotifyKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/square/square.go b/pkg/detectors/square/square.go
new file mode 100644
index 000000000..e2f215f14
--- /dev/null
+++ b/pkg/detectors/square/square.go
@@ -0,0 +1,89 @@
+package square
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ // more context to be added if this is too generic
+ secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"square"}) + `(EAAA[a-zA-Z0-9\-\+\=]{60})`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"EAAA"}
+}
+
+// FromData will find and optionally verify Square secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ // Surprisingly there are still a lot of false positives! So, also doing substring check for square.
+ if !strings.Contains(strings.ToLower(dataStr), "square") {
+ return
+ }
+
+ secMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+ for _, secMatch := range secMatches {
+ if len(secMatch) != 2 {
+ continue
+ }
+ res := strings.TrimSpace(secMatch[1])
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Square,
+ Raw: []byte(res),
+ }
+
+ if verify {
+ // there are a few endpoints we can check, but templates seems the least sensitive.
+ // 403 will be issued if the scope is wrong but the key is correct
+ baseURL := "https://connect.squareupsandbox.com/v2/merchants"
+
+ client := common.RetryableHttpClient()
+
+ // test `merchants` scope - its commonly allowed and low sensitivity
+ req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", res))
+ req.Header.Add("Content-Type", "application/json")
+ // unclear if this version needs to be set or matters, seems to work without, but docs want it
+ //req.Header.Add("Square-Version", "2020-08-12")
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ // 200 means good key and has `merchants` scope - default allowed by square
+ // 401 is bad key
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/square/square_test.go b/pkg/detectors/square/square_test.go
new file mode 100644
index 000000000..8b9778604
--- /dev/null
+++ b/pkg/detectors/square/square_test.go
@@ -0,0 +1,112 @@
+package square
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestSquare_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SQUARE_SECRET")
+ secretInactive := testSecrets.MustGetField("SQUARE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unvierifed",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Square.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Square.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/squareapp/squareapp.go b/pkg/detectors/squareapp/squareapp.go
new file mode 100644
index 000000000..fe2caedca
--- /dev/null
+++ b/pkg/detectors/squareapp/squareapp.go
@@ -0,0 +1,96 @@
+package squareapp
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ //possibly always `sq0csp` for secret
+ // and `sq0idb` for app
+ keyPat = regexp.MustCompile(`[\w\-]*sq0i[a-z]{2}-[0-9A-Za-z\-_]{22,43}`)
+ secPat = regexp.MustCompile(`[\w\-]*sq0c[a-z]{2}-[0-9A-Za-z\-_]{40,50}`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"sq0i"}
+}
+
+// FromData will find and optionally verify SquareApp secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllString(dataStr, -1)
+ secMatches := secPat.FindAllString(dataStr, -1)
+ for _, match := range matches {
+
+ for _, secMatch := range secMatches {
+
+ if detectors.IsKnownFalsePositive(strings.ToLower(secMatch), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Square,
+ Raw: []byte(match),
+ Redacted: match,
+ }
+
+ if verify {
+ baseURL := "https://connect.squareupsandbox.com/oauth2/revoke"
+
+ client := common.RetryableHttpClient()
+ reqData, err := json.Marshal(map[string]string{
+ "client_id": match,
+ "access_token": "fakeTruffleHogAccessTokenForVerification",
+ })
+ if err != nil {
+ return results, err
+ }
+
+ req, _ := http.NewRequestWithContext(ctx, "POST", baseURL, bytes.NewReader(reqData))
+ req.Header.Add("Authorization", fmt.Sprintf("Client %s", secMatch))
+ req.Header.Add("Content-Type", "application/json")
+
+ // unclear if this version needs to be set or matters, seems to work without, but docs want it
+ //req.Header.Add("Square-Version", "2020-08-12")
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ // 404 = Correct crentials. The fake access token should not be found
+ if res.StatusCode == http.StatusNotFound {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+ }
+ return
+}
diff --git a/pkg/detectors/squareapp/squareapp_test.go b/pkg/detectors/squareapp/squareapp_test.go
new file mode 100644
index 000000000..10d4a7e20
--- /dev/null
+++ b/pkg/detectors/squareapp/squareapp_test.go
@@ -0,0 +1,114 @@
+package squareapp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestSquareApp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SQUAREAPP_SECRET")
+ secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")
+ id := testSecrets.MustGetField("SQUAREAPP_ID")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: true,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: false,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SquareApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SquareApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/squarespace/squarespace_test.go b/pkg/detectors/squarespace/squarespace_test.go
index 2b3e5a3d5..2858518d4 100644
--- a/pkg/detectors/squarespace/squarespace_test.go
+++ b/pkg/detectors/squarespace/squarespace_test.go
@@ -16,7 +16,7 @@ import (
func TestSquarespace_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/squareup/squareup_test.go b/pkg/detectors/squareup/squareup_test.go
index 4594ff46a..796f27fe6 100644
--- a/pkg/detectors/squareup/squareup_test.go
+++ b/pkg/detectors/squareup/squareup_test.go
@@ -16,7 +16,7 @@ import (
func TestSquareup_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sslmate/sslmate_test.go b/pkg/detectors/sslmate/sslmate_test.go
index 0e40efec5..c70c7ec99 100644
--- a/pkg/detectors/sslmate/sslmate_test.go
+++ b/pkg/detectors/sslmate/sslmate_test.go
@@ -16,7 +16,7 @@ import (
func TestSslMate_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/stitchdata/stitchdata_test.go b/pkg/detectors/stitchdata/stitchdata_test.go
index a421a9d2d..4f6f591cf 100644
--- a/pkg/detectors/stitchdata/stitchdata_test.go
+++ b/pkg/detectors/stitchdata/stitchdata_test.go
@@ -16,7 +16,7 @@ import (
func TestStitchdata_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/stockdata/stockdata_test.go b/pkg/detectors/stockdata/stockdata_test.go
index 3b7812c12..0d6b52daa 100644
--- a/pkg/detectors/stockdata/stockdata_test.go
+++ b/pkg/detectors/stockdata/stockdata_test.go
@@ -16,7 +16,7 @@ import (
func TestStockdata_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/storecove/storecove_test.go b/pkg/detectors/storecove/storecove_test.go
index acadb3083..8b4f425b9 100644
--- a/pkg/detectors/storecove/storecove_test.go
+++ b/pkg/detectors/storecove/storecove_test.go
@@ -16,7 +16,7 @@ import (
func TestStorecove_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/stormglass/stormglass_test.go b/pkg/detectors/stormglass/stormglass_test.go
index 92775d2c3..e5519543b 100644
--- a/pkg/detectors/stormglass/stormglass_test.go
+++ b/pkg/detectors/stormglass/stormglass_test.go
@@ -16,7 +16,7 @@ import (
func TestStormglass_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/storyblok/storyblok_test.go b/pkg/detectors/storyblok/storyblok_test.go
index a20e46159..6b413e313 100644
--- a/pkg/detectors/storyblok/storyblok_test.go
+++ b/pkg/detectors/storyblok/storyblok_test.go
@@ -16,7 +16,7 @@ import (
func TestStoryblok_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/storychief/storychief_test.go b/pkg/detectors/storychief/storychief_test.go
index 6d0ba5ad0..2b395c98a 100644
--- a/pkg/detectors/storychief/storychief_test.go
+++ b/pkg/detectors/storychief/storychief_test.go
@@ -16,7 +16,7 @@ import (
func TestStorychief_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/strava/strava_test.go b/pkg/detectors/strava/strava_test.go
index fb78bb901..0364b11a6 100644
--- a/pkg/detectors/strava/strava_test.go
+++ b/pkg/detectors/strava/strava_test.go
@@ -16,7 +16,7 @@ import (
func TestStrava_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/streak/streak_test.go b/pkg/detectors/streak/streak_test.go
index 3d4540d95..dc1fa4bdd 100644
--- a/pkg/detectors/streak/streak_test.go
+++ b/pkg/detectors/streak/streak_test.go
@@ -16,7 +16,7 @@ import (
func TestStreak_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/stripe/stripe.go b/pkg/detectors/stripe/stripe.go
new file mode 100644
index 000000000..af9c6b1af
--- /dev/null
+++ b/pkg/detectors/stripe/stripe.go
@@ -0,0 +1,76 @@
+package stripe
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ //doesnt include test keys with "sk_test"
+ secretKey = regexp.MustCompile(`[rs]k_live_[a-zA-Z0-9]{20,30}`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"k_live"}
+}
+
+// FromData will find and optionally verify Stripe secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+
+ dataStr := string(data)
+
+ matches := secretKey.FindAllString(dataStr, -1)
+
+ for _, match := range matches {
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Stripe,
+ Raw: []byte(match),
+ }
+
+ if verify {
+
+ baseURL := "https://api.stripe.com/v1/charges"
+
+ client := common.RetryableHttpClient()
+
+ // test `read_user` scope
+ req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match))
+ req.Header.Add("Content-Type", "application/json")
+ res, err := client.Do(req)
+ if err != nil {
+ return results, err
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/stripe/stripe_test.go b/pkg/detectors/stripe/stripe_test.go
new file mode 100644
index 000000000..ad83d629c
--- /dev/null
+++ b/pkg/detectors/stripe/stripe_test.go
@@ -0,0 +1,111 @@
+package stripe
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestStripe_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STRIPE_SECRET")
+ secretInactive := testSecrets.MustGetField("STRIPE_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stripe,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stripe,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stripe.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stripe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stytch/stytch_test.go b/pkg/detectors/stytch/stytch_test.go
index 6945e315b..2c1b0d1e9 100644
--- a/pkg/detectors/stytch/stytch_test.go
+++ b/pkg/detectors/stytch/stytch_test.go
@@ -16,7 +16,7 @@ import (
func TestStytch_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/sumologickey/sumologickey_test.go b/pkg/detectors/sumologickey/sumologickey_test.go
index c3c8ef5d4..768051b41 100644
--- a/pkg/detectors/sumologickey/sumologickey_test.go
+++ b/pkg/detectors/sumologickey/sumologickey_test.go
@@ -16,7 +16,7 @@ import (
func TestSumoLogicKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/surveyanyplace/surveyanyplace_test.go b/pkg/detectors/surveyanyplace/surveyanyplace_test.go
index 4085c3f92..a31ba1687 100644
--- a/pkg/detectors/surveyanyplace/surveyanyplace_test.go
+++ b/pkg/detectors/surveyanyplace/surveyanyplace_test.go
@@ -16,7 +16,7 @@ import (
func TestSurveyAnyplace_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/surveybot/surveybot_test.go b/pkg/detectors/surveybot/surveybot_test.go
index b2164f547..9ffe87c36 100644
--- a/pkg/detectors/surveybot/surveybot_test.go
+++ b/pkg/detectors/surveybot/surveybot_test.go
@@ -16,7 +16,7 @@ import (
func TestSurveyBot_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/surveysparrow/surveysparrow_test.go b/pkg/detectors/surveysparrow/surveysparrow_test.go
index a44889f50..40778b0bd 100644
--- a/pkg/detectors/surveysparrow/surveysparrow_test.go
+++ b/pkg/detectors/surveysparrow/surveysparrow_test.go
@@ -16,7 +16,7 @@ import (
func TestSurveySparrow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/survicate/survicate_test.go b/pkg/detectors/survicate/survicate_test.go
index 78cd23a7f..388151dbe 100644
--- a/pkg/detectors/survicate/survicate_test.go
+++ b/pkg/detectors/survicate/survicate_test.go
@@ -16,7 +16,7 @@ import (
func TestSurvicate_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/swell/swell_test.go b/pkg/detectors/swell/swell_test.go
index 9922472eb..9a9ce655e 100644
--- a/pkg/detectors/swell/swell_test.go
+++ b/pkg/detectors/swell/swell_test.go
@@ -16,7 +16,7 @@ import (
func TestSwell_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/swiftype/swiftype_test.go b/pkg/detectors/swiftype/swiftype_test.go
index e5abab3ab..d9714abe3 100644
--- a/pkg/detectors/swiftype/swiftype_test.go
+++ b/pkg/detectors/swiftype/swiftype_test.go
@@ -16,7 +16,7 @@ import (
func TestSwiftype_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tatumio/tatumio_test.go b/pkg/detectors/tatumio/tatumio_test.go
index 88f72e600..ccf01bf55 100644
--- a/pkg/detectors/tatumio/tatumio_test.go
+++ b/pkg/detectors/tatumio/tatumio_test.go
@@ -16,7 +16,7 @@ import (
func TestTatumIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/taxjar/taxjar_test.go b/pkg/detectors/taxjar/taxjar_test.go
index 4b7f20f73..94a88c925 100644
--- a/pkg/detectors/taxjar/taxjar_test.go
+++ b/pkg/detectors/taxjar/taxjar_test.go
@@ -16,7 +16,7 @@ import (
func TestTaxjar_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/teamgate/teamgate_test.go b/pkg/detectors/teamgate/teamgate_test.go
index 8a91aa723..d1bf87e50 100644
--- a/pkg/detectors/teamgate/teamgate_test.go
+++ b/pkg/detectors/teamgate/teamgate_test.go
@@ -16,7 +16,7 @@ import (
func TestTeamgate_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go b/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go
index c98d39e2e..ec84b4594 100644
--- a/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go
+++ b/pkg/detectors/technicalanalysisapi/technicalanalysisapi_test.go
@@ -16,7 +16,7 @@ import (
func TestTechnicalAnalysisApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/telegrambottoken/telegrambottoken_test.go b/pkg/detectors/telegrambottoken/telegrambottoken_test.go
index 618be7451..f420e307e 100644
--- a/pkg/detectors/telegrambottoken/telegrambottoken_test.go
+++ b/pkg/detectors/telegrambottoken/telegrambottoken_test.go
@@ -16,7 +16,7 @@ import (
func TestTelegramBotToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/telnyx/telnyx_test.go b/pkg/detectors/telnyx/telnyx_test.go
index 75097a41e..eb6cb4669 100644
--- a/pkg/detectors/telnyx/telnyx_test.go
+++ b/pkg/detectors/telnyx/telnyx_test.go
@@ -16,7 +16,7 @@ import (
func TestTelnyx_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go b/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go
index 070b193fa..f9ca6100a 100644
--- a/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go
+++ b/pkg/detectors/terraformcloudpersonaltoken/terraformcloudpersonaltoken_test.go
@@ -16,7 +16,7 @@ import (
func TestTerraformCloudPersonalToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/text2data/text2data_test.go b/pkg/detectors/text2data/text2data_test.go
index c8d75037f..706597554 100644
--- a/pkg/detectors/text2data/text2data_test.go
+++ b/pkg/detectors/text2data/text2data_test.go
@@ -16,7 +16,7 @@ import (
func TestText2Data_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/textmagic/textmagic_test.go b/pkg/detectors/textmagic/textmagic_test.go
index 3ffefa840..f21e6fdfd 100644
--- a/pkg/detectors/textmagic/textmagic_test.go
+++ b/pkg/detectors/textmagic/textmagic_test.go
@@ -16,7 +16,7 @@ import (
func TestTextmagic_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/thinkific/thinkific_test.go b/pkg/detectors/thinkific/thinkific_test.go
index 57791f57c..8c5ca9585 100644
--- a/pkg/detectors/thinkific/thinkific_test.go
+++ b/pkg/detectors/thinkific/thinkific_test.go
@@ -16,7 +16,7 @@ import (
func TestThinkific_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/thousandeyes/thousandeyes_test.go b/pkg/detectors/thousandeyes/thousandeyes_test.go
index 41155a76f..a7fda41e4 100644
--- a/pkg/detectors/thousandeyes/thousandeyes_test.go
+++ b/pkg/detectors/thousandeyes/thousandeyes_test.go
@@ -16,7 +16,7 @@ import (
func TestThousandEyes_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ticketmaster/ticketmaster_test.go b/pkg/detectors/ticketmaster/ticketmaster_test.go
index be670d586..95435d318 100644
--- a/pkg/detectors/ticketmaster/ticketmaster_test.go
+++ b/pkg/detectors/ticketmaster/ticketmaster_test.go
@@ -16,7 +16,7 @@ import (
func TestTicketMaster_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tiingo/tiingo.go b/pkg/detectors/tiingo/tiingo.go
new file mode 100644
index 000000000..20e8af23b
--- /dev/null
+++ b/pkg/detectors/tiingo/tiingo.go
@@ -0,0 +1,75 @@
+package tiingo
+
+import (
+ "context"
+ "fmt"
+
+ // "log"
+ "regexp"
+ "strings"
+
+ "net/http"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ client = common.SaneHttpClient()
+
+ //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"tiingo"}) + `\b([0-9a-z]{40})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"tiingo"}
+}
+
+// FromData will find and optionally verify Tiingo secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+ if len(match) != 2 {
+ continue
+ }
+ resMatch := strings.TrimSpace(match[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Tiingo,
+ Raw: []byte(resMatch),
+ }
+
+ if verify {
+ req, _ := http.NewRequest("GET", "https://api.tiingo.com/tiingo/fundamentals/definitions", nil)
+ req.Header.Add("Content-Type", "application/json")
+ req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s1.Verified = true
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+ }
+ }
+
+ results = append(results, s1)
+ }
+
+ return detectors.CleanResults(results), nil
+}
diff --git a/pkg/detectors/tiingo/tiingo_test.go b/pkg/detectors/tiingo/tiingo_test.go
new file mode 100644
index 000000000..4fa578f9a
--- /dev/null
+++ b/pkg/detectors/tiingo/tiingo_test.go
@@ -0,0 +1,113 @@
+package tiingo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestTiingo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("TIINGO")
+ inactiveSecret := testSecrets.MustGetField("TIINGO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a tiingo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Tiingo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a tiingo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Tiingo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Tiingo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Tiingo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/timezoneapi/timezoneapi_test.go b/pkg/detectors/timezoneapi/timezoneapi_test.go
index 2504ed2aa..63a0446be 100644
--- a/pkg/detectors/timezoneapi/timezoneapi_test.go
+++ b/pkg/detectors/timezoneapi/timezoneapi_test.go
@@ -16,7 +16,7 @@ import (
func TestTimezoneapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tly/tly_test.go b/pkg/detectors/tly/tly_test.go
index 08c8f5751..f96a43d12 100644
--- a/pkg/detectors/tly/tly_test.go
+++ b/pkg/detectors/tly/tly_test.go
@@ -16,7 +16,7 @@ import (
func TestTLy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tmetric/tmetric_test.go b/pkg/detectors/tmetric/tmetric_test.go
index 0753b4586..3e579a87a 100644
--- a/pkg/detectors/tmetric/tmetric_test.go
+++ b/pkg/detectors/tmetric/tmetric_test.go
@@ -16,7 +16,7 @@ import (
func TestTmetric_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/todoist/todoist_test.go b/pkg/detectors/todoist/todoist_test.go
index b48a8c975..aaa572c2b 100644
--- a/pkg/detectors/todoist/todoist_test.go
+++ b/pkg/detectors/todoist/todoist_test.go
@@ -16,7 +16,7 @@ import (
func TestTodoist_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/toggltrack/toggltrack_test.go b/pkg/detectors/toggltrack/toggltrack_test.go
index 8de05dfe8..4cd0e893a 100644
--- a/pkg/detectors/toggltrack/toggltrack_test.go
+++ b/pkg/detectors/toggltrack/toggltrack_test.go
@@ -16,7 +16,7 @@ import (
func TestTogglTrack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tomorrowio/tomorrowio_test.go b/pkg/detectors/tomorrowio/tomorrowio_test.go
index b4cd59619..f86feb343 100644
--- a/pkg/detectors/tomorrowio/tomorrowio_test.go
+++ b/pkg/detectors/tomorrowio/tomorrowio_test.go
@@ -16,7 +16,7 @@ import (
func TestTomorrowIO_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tomtom/tomtom_test.go b/pkg/detectors/tomtom/tomtom_test.go
index 9b2b2b2d7..0fe2dbd52 100644
--- a/pkg/detectors/tomtom/tomtom_test.go
+++ b/pkg/detectors/tomtom/tomtom_test.go
@@ -16,7 +16,7 @@ import (
func TestTomtom_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tradier/tradier_test.go b/pkg/detectors/tradier/tradier_test.go
index fc69478a1..52abf9271 100644
--- a/pkg/detectors/tradier/tradier_test.go
+++ b/pkg/detectors/tradier/tradier_test.go
@@ -16,7 +16,7 @@ import (
func TestTradier_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/travelpayouts/travelpayouts_test.go b/pkg/detectors/travelpayouts/travelpayouts_test.go
index 8a1823566..c168a791d 100644
--- a/pkg/detectors/travelpayouts/travelpayouts_test.go
+++ b/pkg/detectors/travelpayouts/travelpayouts_test.go
@@ -16,7 +16,7 @@ import (
func TestTravelPayouts_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/travisci/travisci_test.go b/pkg/detectors/travisci/travisci_test.go
index 73289293b..df66b0346 100644
--- a/pkg/detectors/travisci/travisci_test.go
+++ b/pkg/detectors/travisci/travisci_test.go
@@ -16,7 +16,7 @@ import (
func TestTravisCI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/trelloapikey/trelloapikey_test.go b/pkg/detectors/trelloapikey/trelloapikey_test.go
index 2eb55b0bc..7f64f2c9f 100644
--- a/pkg/detectors/trelloapikey/trelloapikey_test.go
+++ b/pkg/detectors/trelloapikey/trelloapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestTrelloApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/tru/tru_test.go b/pkg/detectors/tru/tru_test.go
index be04b152a..747a68e5a 100644
--- a/pkg/detectors/tru/tru_test.go
+++ b/pkg/detectors/tru/tru_test.go
@@ -16,7 +16,7 @@ import (
func TestTru_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/twelvedata/twelvedata_test.go b/pkg/detectors/twelvedata/twelvedata_test.go
index 91d698e48..08a2b64ef 100644
--- a/pkg/detectors/twelvedata/twelvedata_test.go
+++ b/pkg/detectors/twelvedata/twelvedata_test.go
@@ -16,7 +16,7 @@ import (
func TestTwelveData_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.go
new file mode 100644
index 000000000..403f02db0
--- /dev/null
+++ b/pkg/detectors/twilio/twilio.go
@@ -0,0 +1,90 @@
+package twilio
+
+import (
+ "context"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
+ sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
+ keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"sid"}
+}
+
+// FromData will find and optionally verify Twilio secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ identifierMatches := identifierPat.FindAllString(dataStr, -1)
+
+ if len(identifierMatches) == 0 {
+ return
+ }
+
+ keyMatches := keyPat.FindAllString(dataStr, -1)
+ sidMatches := sidPat.FindAllString(dataStr, -1)
+
+ for _, sid := range sidMatches {
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Twilio,
+ Raw: []byte(sid),
+ Redacted: sid,
+ }
+
+ if verify {
+ client := common.RetryableHttpClient()
+ for _, key := range keyMatches {
+
+ form := url.Values{}
+ form.Add("FriendlyName", "MyServiceName")
+ req, _ := http.NewRequestWithContext(
+ ctx, "POST", "https://verify.twilio.com/v2/Services",
+ strings.NewReader(form.Encode()),
+ )
+ req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+ req.Header.Add("Accept", "*/*")
+ req.SetBasicAuth(sid, key)
+ res, err := client.Do(req)
+ if err != nil {
+ break
+ }
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ s.Verified = true
+ }
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ if len(keyMatches) > 0 {
+ results = append(results, s)
+ }
+ }
+
+ return
+}
diff --git a/pkg/detectors/twilio/twilio_test.go b/pkg/detectors/twilio/twilio_test.go
new file mode 100644
index 000000000..17805006f
--- /dev/null
+++ b/pkg/detectors/twilio/twilio_test.go
@@ -0,0 +1,115 @@
+package twilio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestTwilio_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("TWILLIO_API")
+ secretInactive := testSecrets.MustGetField("TWILLIO_API_INACTIVE")
+ id := testSecrets.MustGetField("TWILLIO_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Twilio,
+ Verified: true,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, not verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secretInactive, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Twilio,
+ Verified: false,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/tyntec/tyntec_test.go b/pkg/detectors/tyntec/tyntec_test.go
index d0885e7b2..40968a91c 100644
--- a/pkg/detectors/tyntec/tyntec_test.go
+++ b/pkg/detectors/tyntec/tyntec_test.go
@@ -16,7 +16,7 @@ import (
func TestTyntec_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/typeform/typeform_test.go b/pkg/detectors/typeform/typeform_test.go
index 2d612f5d6..9cc14c102 100644
--- a/pkg/detectors/typeform/typeform_test.go
+++ b/pkg/detectors/typeform/typeform_test.go
@@ -16,7 +16,7 @@ import (
func TestTypeform_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/ubidots/ubidots_test.go b/pkg/detectors/ubidots/ubidots_test.go
index 27b3be212..9ad99cf6e 100644
--- a/pkg/detectors/ubidots/ubidots_test.go
+++ b/pkg/detectors/ubidots/ubidots_test.go
@@ -16,7 +16,7 @@ import (
func TestUbidots_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/unifyid/unifyid_test.go b/pkg/detectors/unifyid/unifyid_test.go
index c47974ba4..f5ebb8bd1 100644
--- a/pkg/detectors/unifyid/unifyid_test.go
+++ b/pkg/detectors/unifyid/unifyid_test.go
@@ -16,7 +16,7 @@ import (
func TestUnifyid_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/unplugg/unplugg_test.go b/pkg/detectors/unplugg/unplugg_test.go
index b80f3e204..269155fc7 100644
--- a/pkg/detectors/unplugg/unplugg_test.go
+++ b/pkg/detectors/unplugg/unplugg_test.go
@@ -16,7 +16,7 @@ import (
func TestUnplugg_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/unsplash/unsplash_test.go b/pkg/detectors/unsplash/unsplash_test.go
index a018d10fc..02d3fa71e 100644
--- a/pkg/detectors/unsplash/unsplash_test.go
+++ b/pkg/detectors/unsplash/unsplash_test.go
@@ -16,7 +16,7 @@ import (
func TestUnsplash_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/upcdatabase/upcdatabase_test.go b/pkg/detectors/upcdatabase/upcdatabase_test.go
index a7120aec7..37f30c851 100644
--- a/pkg/detectors/upcdatabase/upcdatabase_test.go
+++ b/pkg/detectors/upcdatabase/upcdatabase_test.go
@@ -16,7 +16,7 @@ import (
func TestUPCDatabase_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/uplead/uplead_test.go b/pkg/detectors/uplead/uplead_test.go
index 25fd1dfdc..a3bd36648 100644
--- a/pkg/detectors/uplead/uplead_test.go
+++ b/pkg/detectors/uplead/uplead_test.go
@@ -16,7 +16,7 @@ import (
func TestUplead_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/upwave/upwave_test.go b/pkg/detectors/upwave/upwave_test.go
index 4dceb74f3..b62bf1cdd 100644
--- a/pkg/detectors/upwave/upwave_test.go
+++ b/pkg/detectors/upwave/upwave_test.go
@@ -16,7 +16,7 @@ import (
func TestUpwave_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go
new file mode 100644
index 000000000..92bf1d08c
--- /dev/null
+++ b/pkg/detectors/uri/uri.go
@@ -0,0 +1,134 @@
+package uri
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+)
+
+type Scanner struct {
+ allowKnownTestSites bool
+}
+
+// Ensure the Scanner satisfies the interface at compile time
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ keyPat = regexp.MustCompile(`\b[a-zA-Z]{1,10}:?\/\/[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+\b`)
+)
+
+type proxyRes struct {
+ Verified bool `json:"verified"`
+}
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"http"}
+}
+
+func allowlistedProtos(scheme string) bool {
+ allowlisted := []string{"http", "https", "mongodb", "redis", "ftp"}
+ for _, s := range allowlisted {
+ if s == scheme {
+ return true
+ }
+ }
+ return false
+}
+
+// FromData will find and optionally verify URI secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ //Prevent SSRF "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9"
+ //TODO add as config option
+ //TODO extend to other http outbound calls in other areas of code
+ ssrfProtectorURL := "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9"
+
+ dataStr := string(data)
+
+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, match := range matches {
+
+ if !s.allowKnownTestSites {
+ if strings.Contains(match[0], "httpbin.org") {
+ continue
+ }
+ if strings.Contains(match[0], "httpwatch.com") {
+ continue
+ }
+ }
+
+ token := match[0]
+ password := match[1]
+
+ parsedURL, err := url.Parse(token)
+ if err != nil {
+ continue
+ }
+ if _, ok := parsedURL.User.Password(); !ok {
+ continue
+ }
+ if !allowlistedProtos(parsedURL.Scheme) {
+ continue
+ }
+
+ redact := strings.TrimSpace(strings.Replace(token, password, strings.Repeat("*", len(password)), -1))
+
+ s := detectors.Result{
+ DetectorType: detectorspb.DetectorType_URI,
+ Raw: []byte(token),
+ Redacted: redact,
+ }
+
+ if verify {
+ client := common.SaneHttpClient()
+ // whitelist protocols
+
+ // Assume a 200 response is a valid credential
+ postValues := map[string]string{"protocol": parsedURL.Scheme, "credentialed_uri": token}
+ jsonValue, _ := json.Marshal(postValues)
+ req, _ := http.NewRequestWithContext(ctx, "POST", ssrfProtectorURL, bytes.NewBuffer(jsonValue))
+ req.Header.Add("Content-Type", "application/json")
+ res, err := client.Do(req)
+ if err != nil {
+ // log.WithError(err).Warn("Error in http post to SSRF proxy")
+ continue
+ }
+ defer res.Body.Close()
+ result := proxyRes{}
+ body, err := ioutil.ReadAll(res.Body)
+ if len(body) == 0 || err != nil {
+ continue
+ }
+ err = json.Unmarshal(body, &result)
+ if err != nil {
+ // log.WithField("body", string(body)).WithError(err).Debug("Error decoding SSRF proxy response")
+ continue
+ }
+ if result.Verified {
+ s.Verified = true
+ }
+ }
+
+ if !s.Verified {
+ if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+ continue
+ }
+ }
+
+ results = append(results, s)
+ }
+
+ return
+}
diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go
new file mode 100644
index 000000000..8630076eb
--- /dev/null
+++ b/pkg/detectors/uri/uri_test.go
@@ -0,0 +1,113 @@
+package uri
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/detectorspb"
+)
+
+func TestURI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("URI_SECRET")
+ secretInactive := testSecrets.MustGetField("URI_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+
+ {
+ name: "found, unverified, wrong username",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_URI,
+ Verified: false,
+ Redacted: "https://user:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_URI,
+ Verified: true,
+ Redacted: "https://httpwatch:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "bad proto",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("file://user:[email protected]:123/wh/at/ever"),
+ verify: true,
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{allowKnownTestSites: true}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ // if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ // return
+ // }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/urlscan/urlscan_test.go b/pkg/detectors/urlscan/urlscan_test.go
index 99bb99dbc..bccc3a37a 100644
--- a/pkg/detectors/urlscan/urlscan_test.go
+++ b/pkg/detectors/urlscan/urlscan_test.go
@@ -16,7 +16,7 @@ import (
func TestUrlscan_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/user/user_test.go b/pkg/detectors/user/user_test.go
index 59de722ba..b126135d2 100644
--- a/pkg/detectors/user/user_test.go
+++ b/pkg/detectors/user/user_test.go
@@ -16,7 +16,7 @@ import (
func TestUser_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/userstack/userstack_test.go b/pkg/detectors/userstack/userstack_test.go
index ed11cfea9..e45228731 100644
--- a/pkg/detectors/userstack/userstack_test.go
+++ b/pkg/detectors/userstack/userstack_test.go
@@ -16,7 +16,7 @@ import (
func TestUserStack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/vatlayer/vatlayer_test.go b/pkg/detectors/vatlayer/vatlayer_test.go
index 17a74ed97..be0079217 100644
--- a/pkg/detectors/vatlayer/vatlayer_test.go
+++ b/pkg/detectors/vatlayer/vatlayer_test.go
@@ -16,7 +16,7 @@ import (
func TestVatLayer_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/vercel/vercel_test.go b/pkg/detectors/vercel/vercel_test.go
index 67f541e8c..8f32e6aa9 100644
--- a/pkg/detectors/vercel/vercel_test.go
+++ b/pkg/detectors/vercel/vercel_test.go
@@ -16,7 +16,7 @@ import (
func TestVercel_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/verifier/verifier_test.go b/pkg/detectors/verifier/verifier_test.go
index c0ee0a7d9..85bf48d6a 100644
--- a/pkg/detectors/verifier/verifier_test.go
+++ b/pkg/detectors/verifier/verifier_test.go
@@ -16,7 +16,7 @@ import (
func TestVerifier_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/verimail/verimail_test.go b/pkg/detectors/verimail/verimail_test.go
index 758aad8e0..564062f5d 100644
--- a/pkg/detectors/verimail/verimail_test.go
+++ b/pkg/detectors/verimail/verimail_test.go
@@ -16,7 +16,7 @@ import (
func TestVerimail_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/veriphone/veriphone_test.go b/pkg/detectors/veriphone/veriphone_test.go
index 28b86985e..6309dd278 100644
--- a/pkg/detectors/veriphone/veriphone_test.go
+++ b/pkg/detectors/veriphone/veriphone_test.go
@@ -16,7 +16,7 @@ import (
func TestVeriphone_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/versioneye/versioneye_test.go b/pkg/detectors/versioneye/versioneye_test.go
index 6d915d9bb..614cecc73 100644
--- a/pkg/detectors/versioneye/versioneye_test.go
+++ b/pkg/detectors/versioneye/versioneye_test.go
@@ -16,7 +16,7 @@ import (
func TestVersionEye_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/viewneo/viewneo_test.go b/pkg/detectors/viewneo/viewneo_test.go
index c4feb49f4..8f2eaa938 100644
--- a/pkg/detectors/viewneo/viewneo_test.go
+++ b/pkg/detectors/viewneo/viewneo_test.go
@@ -16,7 +16,7 @@ import (
func TestViewneo_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/virustotal/virustotal_test.go b/pkg/detectors/virustotal/virustotal_test.go
index a6987111c..a38c72c23 100644
--- a/pkg/detectors/virustotal/virustotal_test.go
+++ b/pkg/detectors/virustotal/virustotal_test.go
@@ -16,7 +16,7 @@ import (
func TestVirusTotal_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/visualcrossing/visualcrossing_test.go b/pkg/detectors/visualcrossing/visualcrossing_test.go
index 34477b634..15f279f1c 100644
--- a/pkg/detectors/visualcrossing/visualcrossing_test.go
+++ b/pkg/detectors/visualcrossing/visualcrossing_test.go
@@ -16,7 +16,7 @@ import (
func TestVisualcrossing_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/voicegain/voicegain_test.go b/pkg/detectors/voicegain/voicegain_test.go
index 3c38cedfe..75467325f 100644
--- a/pkg/detectors/voicegain/voicegain_test.go
+++ b/pkg/detectors/voicegain/voicegain_test.go
@@ -16,7 +16,7 @@ import (
func TestVoicegain_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/vouchery/vouchery_test.go b/pkg/detectors/vouchery/vouchery_test.go
index aae9f8588..0926d12f4 100644
--- a/pkg/detectors/vouchery/vouchery_test.go
+++ b/pkg/detectors/vouchery/vouchery_test.go
@@ -16,7 +16,7 @@ import (
func TestVouchery_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/vpnapi/vpnapi_test.go b/pkg/detectors/vpnapi/vpnapi_test.go
index c7b83c6a8..17a3d9d31 100644
--- a/pkg/detectors/vpnapi/vpnapi_test.go
+++ b/pkg/detectors/vpnapi/vpnapi_test.go
@@ -16,7 +16,7 @@ import (
func TestVpnapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/vultrapikey/vultrapikey_test.go b/pkg/detectors/vultrapikey/vultrapikey_test.go
index 4b784ee0c..2ffc5079c 100644
--- a/pkg/detectors/vultrapikey/vultrapikey_test.go
+++ b/pkg/detectors/vultrapikey/vultrapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestVultrApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/vyte/vyte_test.go b/pkg/detectors/vyte/vyte_test.go
index 9d24c878b..a89a38620 100644
--- a/pkg/detectors/vyte/vyte_test.go
+++ b/pkg/detectors/vyte/vyte_test.go
@@ -16,7 +16,7 @@ import (
func TestVyte_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/walkscore/walkscore_test.go b/pkg/detectors/walkscore/walkscore_test.go
index d0b7671b2..be6881e66 100644
--- a/pkg/detectors/walkscore/walkscore_test.go
+++ b/pkg/detectors/walkscore/walkscore_test.go
@@ -16,7 +16,7 @@ import (
func TestWalkscore_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/weatherbit/weatherbit_test.go b/pkg/detectors/weatherbit/weatherbit_test.go
index cad278ae6..cce0ea7ea 100644
--- a/pkg/detectors/weatherbit/weatherbit_test.go
+++ b/pkg/detectors/weatherbit/weatherbit_test.go
@@ -16,7 +16,7 @@ import (
func TestWeatherbit_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/weatherstack/weatherstack_test.go b/pkg/detectors/weatherstack/weatherstack_test.go
index aa81491b8..5599e738c 100644
--- a/pkg/detectors/weatherstack/weatherstack_test.go
+++ b/pkg/detectors/weatherstack/weatherstack_test.go
@@ -16,7 +16,7 @@ import (
func TestWeatherstack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/webex/webex_test.go b/pkg/detectors/webex/webex_test.go
index 70a139aba..77f21a2db 100644
--- a/pkg/detectors/webex/webex_test.go
+++ b/pkg/detectors/webex/webex_test.go
@@ -16,7 +16,7 @@ import (
func TestWebex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/webflow/webflow_test.go b/pkg/detectors/webflow/webflow_test.go
index c10c4ed86..455760ea9 100644
--- a/pkg/detectors/webflow/webflow_test.go
+++ b/pkg/detectors/webflow/webflow_test.go
@@ -16,7 +16,7 @@ import (
func TestWebflow_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/webscraper/webscraper_test.go b/pkg/detectors/webscraper/webscraper_test.go
index f8ce45dbf..e806d1b72 100644
--- a/pkg/detectors/webscraper/webscraper_test.go
+++ b/pkg/detectors/webscraper/webscraper_test.go
@@ -16,7 +16,7 @@ import (
func TestWebScraper_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/webscraping/webscraping_test.go b/pkg/detectors/webscraping/webscraping_test.go
index 8c3b12aa1..dcacdb434 100644
--- a/pkg/detectors/webscraping/webscraping_test.go
+++ b/pkg/detectors/webscraping/webscraping_test.go
@@ -16,7 +16,7 @@ import (
func TestWebscraping_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/wepay/wepay_test.go b/pkg/detectors/wepay/wepay_test.go
index df71bfcb8..c8a5709b7 100644
--- a/pkg/detectors/wepay/wepay_test.go
+++ b/pkg/detectors/wepay/wepay_test.go
@@ -16,7 +16,7 @@ import (
func TestWePay_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/whoxy/whoxy_test.go b/pkg/detectors/whoxy/whoxy_test.go
index adafc5a92..a044a3a4f 100644
--- a/pkg/detectors/whoxy/whoxy_test.go
+++ b/pkg/detectors/whoxy/whoxy_test.go
@@ -16,7 +16,7 @@ import (
func TestWhoxy_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/worksnaps/worksnaps_test.go b/pkg/detectors/worksnaps/worksnaps_test.go
index 46ed8a332..bf2895d80 100644
--- a/pkg/detectors/worksnaps/worksnaps_test.go
+++ b/pkg/detectors/worksnaps/worksnaps_test.go
@@ -16,7 +16,7 @@ import (
func TestWorksnaps_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/workstack/workstack_test.go b/pkg/detectors/workstack/workstack_test.go
index fc3049c9c..4db3c8b3e 100644
--- a/pkg/detectors/workstack/workstack_test.go
+++ b/pkg/detectors/workstack/workstack_test.go
@@ -16,7 +16,7 @@ import (
func TestWorkstack_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/worldcoinindex/worldcoinindex_test.go b/pkg/detectors/worldcoinindex/worldcoinindex_test.go
index 3746ee31c..f8e78eed1 100644
--- a/pkg/detectors/worldcoinindex/worldcoinindex_test.go
+++ b/pkg/detectors/worldcoinindex/worldcoinindex_test.go
@@ -16,7 +16,7 @@ import (
func TestWorldCoinIndex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/worldweather/worldweather_test.go b/pkg/detectors/worldweather/worldweather_test.go
index 182fd2f96..a785243fa 100644
--- a/pkg/detectors/worldweather/worldweather_test.go
+++ b/pkg/detectors/worldweather/worldweather_test.go
@@ -16,7 +16,7 @@ import (
func TestWorldweather_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/wrike/wrike_test.go b/pkg/detectors/wrike/wrike_test.go
index d645c7436..62fa95655 100644
--- a/pkg/detectors/wrike/wrike_test.go
+++ b/pkg/detectors/wrike/wrike_test.go
@@ -16,7 +16,7 @@ import (
func TestWrike_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/yandex/yandex_test.go b/pkg/detectors/yandex/yandex_test.go
index cad8c877a..c7b09eb30 100644
--- a/pkg/detectors/yandex/yandex_test.go
+++ b/pkg/detectors/yandex/yandex_test.go
@@ -16,7 +16,7 @@ import (
func TestYandex_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/youneedabudget/youneedabudget_test.go b/pkg/detectors/youneedabudget/youneedabudget_test.go
index e34dba1c2..029426432 100644
--- a/pkg/detectors/youneedabudget/youneedabudget_test.go
+++ b/pkg/detectors/youneedabudget/youneedabudget_test.go
@@ -16,7 +16,7 @@ import (
func TestYouNeedABudget_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/yousign/yousign_test.go b/pkg/detectors/yousign/yousign_test.go
index cbd04534b..e78b89f39 100644
--- a/pkg/detectors/yousign/yousign_test.go
+++ b/pkg/detectors/yousign/yousign_test.go
@@ -16,7 +16,7 @@ import (
func TestYousign_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/youtubeapikey/youtubeapikey_test.go b/pkg/detectors/youtubeapikey/youtubeapikey_test.go
index 2a2326177..1ed4bae88 100644
--- a/pkg/detectors/youtubeapikey/youtubeapikey_test.go
+++ b/pkg/detectors/youtubeapikey/youtubeapikey_test.go
@@ -16,7 +16,7 @@ import (
func TestYoutubeApiKey_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zapierwebhook/zapierwebhook_test.go b/pkg/detectors/zapierwebhook/zapierwebhook_test.go
index cc073f18b..3247128bd 100644
--- a/pkg/detectors/zapierwebhook/zapierwebhook_test.go
+++ b/pkg/detectors/zapierwebhook/zapierwebhook_test.go
@@ -16,7 +16,7 @@ import (
func TestZapierWebhook_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zendeskapi/zendeskapi_test.go b/pkg/detectors/zendeskapi/zendeskapi_test.go
index f79c37843..e6e35d9b7 100644
--- a/pkg/detectors/zendeskapi/zendeskapi_test.go
+++ b/pkg/detectors/zendeskapi/zendeskapi_test.go
@@ -16,7 +16,7 @@ import (
func TestZendeskApi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zenscrape/zenscrape_test.go b/pkg/detectors/zenscrape/zenscrape_test.go
index e052e4dbd..745d38df4 100644
--- a/pkg/detectors/zenscrape/zenscrape_test.go
+++ b/pkg/detectors/zenscrape/zenscrape_test.go
@@ -16,7 +16,7 @@ import (
func TestZenscrape_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zenserp/zenserp_test.go b/pkg/detectors/zenserp/zenserp_test.go
index addddc955..6e08374c6 100644
--- a/pkg/detectors/zenserp/zenserp_test.go
+++ b/pkg/detectors/zenserp/zenserp_test.go
@@ -16,7 +16,7 @@ import (
func TestZenserp_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zeplin/zeplin_test.go b/pkg/detectors/zeplin/zeplin_test.go
index 5b85c41bc..d9f3d9c3e 100644
--- a/pkg/detectors/zeplin/zeplin_test.go
+++ b/pkg/detectors/zeplin/zeplin_test.go
@@ -16,7 +16,7 @@ import (
func TestZeplin_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zerobounce/zerobounce_test.go b/pkg/detectors/zerobounce/zerobounce_test.go
index 091eb8f72..d3aa61255 100644
--- a/pkg/detectors/zerobounce/zerobounce_test.go
+++ b/pkg/detectors/zerobounce/zerobounce_test.go
@@ -16,7 +16,7 @@ import (
func TestZerobounce_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zipapi/zipapi_test.go b/pkg/detectors/zipapi/zipapi_test.go
index adc601d27..59589a9a4 100644
--- a/pkg/detectors/zipapi/zipapi_test.go
+++ b/pkg/detectors/zipapi/zipapi_test.go
@@ -16,7 +16,7 @@ import (
func TestZipapi_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zipbooks/zipbooks_test.go b/pkg/detectors/zipbooks/zipbooks_test.go
index 1606dfc2d..5dadb1dd5 100644
--- a/pkg/detectors/zipbooks/zipbooks_test.go
+++ b/pkg/detectors/zipbooks/zipbooks_test.go
@@ -16,7 +16,7 @@ import (
func TestZipbooks_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 2)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zipcodeapi/zipcodeapi_test.go b/pkg/detectors/zipcodeapi/zipcodeapi_test.go
index 8558c5f19..7d7ea87c7 100644
--- a/pkg/detectors/zipcodeapi/zipcodeapi_test.go
+++ b/pkg/detectors/zipcodeapi/zipcodeapi_test.go
@@ -16,7 +16,7 @@ import (
func TestZipCodeAPI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecrets(ctx)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners1")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/zonkafeedback/zonkafeedback_test.go b/pkg/detectors/zonkafeedback/zonkafeedback_test.go
index 35214c829..f1b2b88ff 100644
--- a/pkg/detectors/zonkafeedback/zonkafeedback_test.go
+++ b/pkg/detectors/zonkafeedback/zonkafeedback_test.go
@@ -16,7 +16,7 @@ import (
func TestZonkaFeedback_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetScannersTestSecretsVersion(ctx, 3)
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "scanners3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}