Skip to content

Find, verify, and analyze leaked credentials

Added pattern test cases for detectors starting with Alphabet C (#3564)

Kashif Khan6921ffda95b3
diff --git a/pkg/detectors/caflou/caflou_integration_test.go b/pkg/detectors/caflou/caflou_integration_test.go
new file mode 100644
index 000000000..b8eb1bec3
--- /dev/null
+++ b/pkg/detectors/caflou/caflou_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package caflou
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCaflou_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CAFLOU")
+ inactiveSecret := testSecrets.MustGetField("CAFLOU_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a caflou secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Caflou,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a caflou secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Caflou,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Caflou.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Caflou.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/caflou/caflou_test.go b/pkg/detectors/caflou/caflou_test.go
index b8eb1bec3..500a2fca0 100644
--- a/pkg/detectors/caflou/caflou_test.go
+++ b/pkg/detectors/caflou/caflou_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package caflou
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ base_url: "https://api.example.com/instances"
+ api_key: $API_KEY
+ caflou_auth_token: "Bearer b8SQoPKLMCBbwIm0XDzbZiDydUk9qNqqBlKnR5Nouwbjs9cv3D1azAXpiFq9WrfNlwxbCwDL2FWCheXmdYKZkMRZklahJh5NQZZY7Zf220hjGJOtKgFbWxy9xQ9hodQqsOOx9Of30qtTrnRxFPa9wxYkSBn"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "b8SQoPKLMCBbwIm0XDzbZiDydUk9qNqqBlKnR5Nouwbjs9cv3D1azAXpiFq9WrfNlwxbCwDL2FWCheXmdYKZkMRZklahJh5NQZZY7Zf220hjGJOtKgFbWxy9xQ9hodQqsOOx9Of30qtTrnRxFPa9wxYkSBn"
)
-func TestCaflou_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CAFLOU")
- inactiveSecret := testSecrets.MustGetField("CAFLOU_INACTIVE")
+func TestCaflou_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a caflou secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Caflou,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a caflou secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Caflou,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Caflou.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Caflou.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/calendarific/calendarific_integration_test.go b/pkg/detectors/calendarific/calendarific_integration_test.go
new file mode 100644
index 000000000..9b6fadba0
--- /dev/null
+++ b/pkg/detectors/calendarific/calendarific_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package calendarific
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCalendarific_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CALENDARIFIC_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CALENDARIFIC_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a calendarific secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Calendarific,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a calendarific secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Calendarific,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Calendarific.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Calendarific.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/calendarific/calendarific_test.go b/pkg/detectors/calendarific/calendarific_test.go
index 9b6fadba0..37f77a241 100644
--- a/pkg/detectors/calendarific/calendarific_test.go
+++ b/pkg/detectors/calendarific/calendarific_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package calendarific
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ calendarific_api_key: "M9qT0uymrS2FgJ78p9CpLIlFOBLUtmao"
+ base_url: "https://api.calendarific.com/v1/holidays?api_key=$calendarific_api_key"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "M9qT0uymrS2FgJ78p9CpLIlFOBLUtmao"
)
-func TestCalendarific_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CALENDARIFIC_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CALENDARIFIC_INACTIVE")
+func TestCalendarific_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a calendarific secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Calendarific,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a calendarific secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Calendarific,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Calendarific.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Calendarific.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/calendlyapikey/calendlyapikey.go b/pkg/detectors/calendlyapikey/calendlyapikey.go
index 86c1da7a9..f63d96c87 100644
--- a/pkg/detectors/calendlyapikey/calendlyapikey.go
+++ b/pkg/detectors/calendlyapikey/calendlyapikey.go
@@ -3,10 +3,11 @@ package calendlyapikey
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/calendlyapikey/calendlyapikey_integration_test.go b/pkg/detectors/calendlyapikey/calendlyapikey_integration_test.go
new file mode 100644
index 000000000..4f1e88a3a
--- /dev/null
+++ b/pkg/detectors/calendlyapikey/calendlyapikey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package calendlyapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCalendlyApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CALENDLYAPIKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CALENDLYAPIKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CalendlyApiKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CalendlyApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CalendlyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CalendlyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/calendlyapikey/calendlyapikey_test.go b/pkg/detectors/calendlyapikey/calendlyapikey_test.go
index 4f1e88a3a..943632729 100644
--- a/pkg/detectors/calendlyapikey/calendlyapikey_test.go
+++ b/pkg/detectors/calendlyapikey/calendlyapikey_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package calendlyapikey
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ api_key: $API_KEY
+ base_url: "https://api.example.com/v1/user"
+ calendly_auth_token: "Bearer eyJuL_8UF5AiQVfO2xlr_HaoSluHz9u-Q-s1qWDWvycQhR11J9wZTmYfFpKnawuIbKjA4i340DSpYI3d3E-oEZZdcHW4cLd_OASWu-H.eyJuOUikPwjw1RKXYXfcjjeqQwdWzA4uooei_ADIUX3of4UzwTjSaaEzLWMGopW4n9Fma0nINBD1qUp_OtbhuH6dmHyv94IeX-hUYla.A0rTdrx3sJ"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "eyJuL_8UF5AiQVfO2xlr_HaoSluHz9u-Q-s1qWDWvycQhR11J9wZTmYfFpKnawuIbKjA4i340DSpYI3d3E-oEZZdcHW4cLd_OASWu-H.eyJuOUikPwjw1RKXYXfcjjeqQwdWzA4uooei_ADIUX3of4UzwTjSaaEzLWMGopW4n9Fma0nINBD1qUp_OtbhuH6dmHyv94IeX-hUYla.A0rTdrx3sJ"
)
-func TestCalendlyApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CALENDLYAPIKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CALENDLYAPIKEY_INACTIVE")
+func TestCalendlyAPIKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CalendlyApiKey,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CalendlyApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CalendlyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CalendlyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/calorieninja/calorieninja.go b/pkg/detectors/calorieninja/calorieninja.go
index 47f594080..362b55f10 100644
--- a/pkg/detectors/calorieninja/calorieninja.go
+++ b/pkg/detectors/calorieninja/calorieninja.go
@@ -2,10 +2,11 @@ package calorieninja
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/calorieninja/calorieninja_integration_test.go b/pkg/detectors/calorieninja/calorieninja_integration_test.go
new file mode 100644
index 000000000..4b680cb75
--- /dev/null
+++ b/pkg/detectors/calorieninja/calorieninja_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package calorieninja
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCalorieninja_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CALORIENINJA")
+ inactiveSecret := testSecrets.MustGetField("CALORIENINJA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CalorieNinja,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CalorieNinja,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Calorieninja.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Calorieninja.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/calorieninja/calorieninja_test.go b/pkg/detectors/calorieninja/calorieninja_test.go
index 4b680cb75..145c9a069 100644
--- a/pkg/detectors/calorieninja/calorieninja_test.go
+++ b/pkg/detectors/calorieninja/calorieninja_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package calorieninja
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ calorieninja_api_key: "ix1aaifujilTcGEjB67e1EBBRXcr7r9cdChAR5hb"
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "ix1aaifujilTcGEjB67e1EBBRXcr7r9cdChAR5hb"
)
-func TestCalorieninja_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CALORIENINJA")
- inactiveSecret := testSecrets.MustGetField("CALORIENINJA_INACTIVE")
+func TestCalorieNinja_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CalorieNinja,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CalorieNinja,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Calorieninja.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Calorieninja.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/campayn/campayn.go b/pkg/detectors/campayn/campayn.go
index c12aaee76..c39485ced 100644
--- a/pkg/detectors/campayn/campayn.go
+++ b/pkg/detectors/campayn/campayn.go
@@ -2,10 +2,11 @@ package campayn
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/campayn/campayn_integration_test.go b/pkg/detectors/campayn/campayn_integration_test.go
new file mode 100644
index 000000000..7cd07d212
--- /dev/null
+++ b/pkg/detectors/campayn/campayn_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package campayn
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCampayn_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CAMPAYN_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CAMPAYN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a campayn secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Campayn,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a campayn secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Campayn,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Campayn.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Campayn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/campayn/campayn_test.go b/pkg/detectors/campayn/campayn_test.go
index 7cd07d212..c5da83acc 100644
--- a/pkg/detectors/campayn/campayn_test.go
+++ b/pkg/detectors/campayn/campayn_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package campayn
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCampayn_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CAMPAYN_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CAMPAYN_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ campayn_api_key: "z6q8z47eu46wri18ygu6pc68vsizprt83jrlu5ustliyhktzoxbzhf1ycdaka978"
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "z6q8z47eu46wri18ygu6pc68vsizprt83jrlu5ustliyhktzoxbzhf1ycdaka978"
+)
+
+func TestCampayn_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a campayn secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Campayn,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a campayn secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Campayn,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Campayn.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Campayn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cannyio/cannyio_integration_test.go b/pkg/detectors/cannyio/cannyio_integration_test.go
new file mode 100644
index 000000000..7226dcaf8
--- /dev/null
+++ b/pkg/detectors/cannyio/cannyio_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cannyio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCannyIo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CANNYIO_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CANNYIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cannyio secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CannyIo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cannyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CannyIo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CannyIo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CannyIo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cannyio/cannyio_test.go b/pkg/detectors/cannyio/cannyio_test.go
index 7226dcaf8..d4dba11b6 100644
--- a/pkg/detectors/cannyio/cannyio_test.go
+++ b/pkg/detectors/cannyio/cannyio_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cannyio
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ cannyio_api_key: "faiiahli-goke-db0r-oxli-s20dgab9a0iv"
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "faiiahli-goke-db0r-oxli-s20dgab9a0iv"
)
-func TestCannyIo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CANNYIO_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CANNYIO_INACTIVE")
+func TestCannyio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cannyio secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CannyIo,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cannyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CannyIo,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CannyIo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CannyIo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/capsulecrm/capsulecrm.go b/pkg/detectors/capsulecrm/capsulecrm.go
index f915e6932..46f34b157 100644
--- a/pkg/detectors/capsulecrm/capsulecrm.go
+++ b/pkg/detectors/capsulecrm/capsulecrm.go
@@ -3,10 +3,11 @@ package capsulecrm
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/capsulecrm/capsulecrm_integration_test.go b/pkg/detectors/capsulecrm/capsulecrm_integration_test.go
new file mode 100644
index 000000000..c63a12412
--- /dev/null
+++ b/pkg/detectors/capsulecrm/capsulecrm_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package capsulecrm
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCapsuleCRM_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CAPSULECRM")
+ inactiveSecret := testSecrets.MustGetField("CAPSULECRM_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CapsuleCRM,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CapsuleCRM,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CapsuleCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CapsuleCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/capsulecrm/capsulecrm_test.go b/pkg/detectors/capsulecrm/capsulecrm_test.go
index c63a12412..18f502220 100644
--- a/pkg/detectors/capsulecrm/capsulecrm_test.go
+++ b/pkg/detectors/capsulecrm/capsulecrm_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package capsulecrm
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ api_key: ""
+ base_url: "https://api.example.com/v1/user"
+ capsulecrm_auth_token: "Bearer ULeIqU-4ss+YImZYsyjPLSsm.9H.SZJ1v.KxT1D-zbaW6sg5b0R5g=koBH4X62hC"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "ULeIqU-4ss+YImZYsyjPLSsm.9H.SZJ1v.KxT1D-zbaW6sg5b0R5g=koBH4X62hC"
)
-func TestCapsuleCRM_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CAPSULECRM")
- inactiveSecret := testSecrets.MustGetField("CAPSULECRM_INACTIVE")
+func TestCapsulecrm_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CapsuleCRM,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CapsuleCRM,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CapsuleCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CapsuleCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/captaindata/v1/captaindata_integration_test.go b/pkg/detectors/captaindata/v1/captaindata_integration_test.go
new file mode 100644
index 000000000..c77264ade
--- /dev/null
+++ b/pkg/detectors/captaindata/v1/captaindata_integration_test.go
@@ -0,0 +1,118 @@
+//go:build detectors
+// +build detectors
+
+package captaindata
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCaptainData_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ projId := testSecrets.MustGetField("CAPTAINDATA_PROJID")
+ secret := testSecrets.MustGetField("CAPTAINDATA")
+ inactiveSecret := testSecrets.MustGetField("CAPTAINDATA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within", projId, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CaptainData,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within but not valid", projId, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CaptainData,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CaptainData.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CaptainData.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/captaindata/v1/captaindata_test.go b/pkg/detectors/captaindata/v1/captaindata_test.go
index c77264ade..71bc022a8 100644
--- a/pkg/detectors/captaindata/v1/captaindata_test.go
+++ b/pkg/detectors/captaindata/v1/captaindata_test.go
@@ -1,117 +1,83 @@
-//go:build detectors
-// +build detectors
-
package captaindata
import (
"context"
- "fmt"
"testing"
- "time"
-
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCaptainData_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- projId := testSecrets.MustGetField("CAPTAINDATA_PROJID")
- secret := testSecrets.MustGetField("CAPTAINDATA")
- inactiveSecret := testSecrets.MustGetField("CAPTAINDATA_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestCaptainData_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within", projId, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CaptainData,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "typical pattern",
+ input: "captaindata_project = '12345678-1234-1234-1234-123456789012' captaindata_api_key = '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'",
+ want: []string{"12345678-1234-1234-1234-1234567890121234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within but not valid", projId, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CaptainData,
- Verified: false,
- },
+ name: "finds all matches",
+ input: `captaindata_project1 = '12345678-1234-1234-1234-123456789012' captaindata_api_key1 = '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'
+captaindata_project2 = '87654321-4321-4321-4321-210987654321' captaindata_api_key2 = 'fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321'`,
+ want: []string{
+ "12345678-1234-1234-1234-1234567890121234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
+ "12345678-1234-1234-1234-123456789012fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321",
+ "87654321-4321-4321-4321-210987654321fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321",
+ "87654321-4321-4321-4321-2109876543211234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",
},
- wantErr: false,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: "captaindata_project = '123456' captaindata_api_key = '1234567890'",
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CaptainData.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
- got[i].Raw = nil
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CaptainData.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/carboninterface/carboninterface.go b/pkg/detectors/carboninterface/carboninterface.go
index d934c998f..977c86549 100644
--- a/pkg/detectors/carboninterface/carboninterface.go
+++ b/pkg/detectors/carboninterface/carboninterface.go
@@ -3,10 +3,11 @@ package carboninterface
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/carboninterface/carboninterface_integration_test.go b/pkg/detectors/carboninterface/carboninterface_integration_test.go
new file mode 100644
index 000000000..e002caa1c
--- /dev/null
+++ b/pkg/detectors/carboninterface/carboninterface_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package carboninterface
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCarbonInterface_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CARBONINTERFACE")
+ inactiveSecret := testSecrets.MustGetField("CARBONINTERFACE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CarbonInterface,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CarbonInterface,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CarbonInterface.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CarbonInterface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/carboninterface/carboninterface_test.go b/pkg/detectors/carboninterface/carboninterface_test.go
index e002caa1c..ab1b0fe9b 100644
--- a/pkg/detectors/carboninterface/carboninterface_test.go
+++ b/pkg/detectors/carboninterface/carboninterface_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package carboninterface
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ api_key: ""
+ base_url: "https://api.example.com/v1/user"
+ carboninterface_auth_token: "Bearer PkN3gWaSHSIjz188TjD4h"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "PkN3gWaSHSIjz188TjD4h"
)
-func TestCarbonInterface_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CARBONINTERFACE")
- inactiveSecret := testSecrets.MustGetField("CARBONINTERFACE_INACTIVE")
+func TestCarbonInterface_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CarbonInterface,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CarbonInterface,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CarbonInterface.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CarbonInterface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cashboard/cashboard.go b/pkg/detectors/cashboard/cashboard.go
index 65efbee34..364a7d749 100644
--- a/pkg/detectors/cashboard/cashboard.go
+++ b/pkg/detectors/cashboard/cashboard.go
@@ -4,16 +4,17 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/cashboard/cashboard_integration_test.go b/pkg/detectors/cashboard/cashboard_integration_test.go
new file mode 100644
index 000000000..4b4fd5db0
--- /dev/null
+++ b/pkg/detectors/cashboard/cashboard_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package cashboard
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCashboard_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CASHBOARD")
+ user := testSecrets.MustGetField("SCANNER_USERNAME")
+ inactiveSecret := testSecrets.MustGetField("CASHBOARD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s", secret, user)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cashboard,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cashboard,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cashboard.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cashboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cashboard/cashboard_test.go b/pkg/detectors/cashboard/cashboard_test.go
index 4b4fd5db0..e6c20e607 100644
--- a/pkg/detectors/cashboard/cashboard_test.go
+++ b/pkg/detectors/cashboard/cashboard_test.go
@@ -1,121 +1,93 @@
-//go:build detectors
-// +build detectors
-
package cashboard
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCashboard_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CASHBOARD")
- user := testSecrets.MustGetField("SCANNER_USERNAME")
- inactiveSecret := testSecrets.MustGetField("CASHBOARD_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ cashboard_key: "F1A-NEI-HY4-PZK"
+ cashboard_user: "ts7z"
+ auth_type: Basic
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "F1A-NEI-HY4-PZKts7z"
+)
+
+func TestCashBoard_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s", secret, user)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cashboard,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cashboard,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cashboard.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cashboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/caspio/caspio_integration_test.go b/pkg/detectors/caspio/caspio_integration_test.go
new file mode 100644
index 000000000..b285d94cf
--- /dev/null
+++ b/pkg/detectors/caspio/caspio_integration_test.go
@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package caspio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCaspio_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CASPIO")
+ id := testSecrets.MustGetField("CASPIO_ID")
+ subdomain := testSecrets.MustGetField("CASPIO_SUBDOMAIN")
+ inactiveSecret := testSecrets.MustGetField("CASPIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s", secret, id, subdomain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Caspio,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s but not valid", inactiveSecret, id, subdomain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Caspio,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Caspio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Caspio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/caspio/caspio_test.go b/pkg/detectors/caspio/caspio_test.go
index b285d94cf..7d56a91b8 100644
--- a/pkg/detectors/caspio/caspio_test.go
+++ b/pkg/detectors/caspio/caspio_test.go
@@ -1,118 +1,98 @@
-//go:build detectors
-// +build detectors
-
package caspio
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCaspio_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CASPIO")
- id := testSecrets.MustGetField("CASPIO_ID")
- subdomain := testSecrets.MustGetField("CASPIO_SUBDOMAIN")
- inactiveSecret := testSecrets.MustGetField("CASPIO_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ caspio_id: "qye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01asty"
+ caspio_secret: "x5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibe"
+ caspio_domain: xlo0xo2s
+ auth_type: Client Credentials
+ base_url: "https://$caspio_domain.caspio.com/v1/user"
+ auth_token: ""
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secrets = []string{
+ "qye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyqye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyxlo0xo2s",
+ "qye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyx5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibexlo0xo2s",
+ "x5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibeqye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyxlo0xo2s",
+ "x5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibex5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibexlo0xo2s",
}
+)
+
+func TestCaspio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s", secret, id, subdomain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Caspio,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s but not valid", inactiveSecret, id, subdomain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Caspio,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: secrets,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Caspio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Caspio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/censys/censys.go b/pkg/detectors/censys/censys.go
index 485c12c64..43f8f11c6 100644
--- a/pkg/detectors/censys/censys.go
+++ b/pkg/detectors/censys/censys.go
@@ -2,10 +2,11 @@ package censys
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/censys/censys_integration_test.go b/pkg/detectors/censys/censys_integration_test.go
new file mode 100644
index 000000000..1b20923c4
--- /dev/null
+++ b/pkg/detectors/censys/censys_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package censys
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCensys_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CENSYS")
+ id := testSecrets.MustGetField("CENSYS_ID")
+ inactiveSecret := testSecrets.MustGetField("CENSYS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Censys,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Censys,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Censys.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Censys.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/censys/censys_test.go b/pkg/detectors/censys/censys_test.go
index 1b20923c4..697004fef 100644
--- a/pkg/detectors/censys/censys_test.go
+++ b/pkg/detectors/censys/censys_test.go
@@ -1,121 +1,93 @@
-//go:build detectors
-// +build detectors
-
package censys
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ censys_key: "roLtT01S6znCRhgoSwNNKY8O7AELn8e4"
+ censys_user: "p4cuaz9fuonwmbfkc4di5uqsizp4yyttpu-q"
+ auth_type: Basic
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "roLtT01S6znCRhgoSwNNKY8O7AELn8e4p4cuaz9fuonwmbfkc4di5uqsizp4yyttpu-q"
)
-func TestCensys_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CENSYS")
- id := testSecrets.MustGetField("CENSYS_ID")
- inactiveSecret := testSecrets.MustGetField("CENSYS_INACTIVE")
+func TestCensys_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Censys,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Censys,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Censys.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Censys.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/centralstationcrm/centralstationcrm.go b/pkg/detectors/centralstationcrm/centralstationcrm.go
index 2d20d6630..a32a7f953 100644
--- a/pkg/detectors/centralstationcrm/centralstationcrm.go
+++ b/pkg/detectors/centralstationcrm/centralstationcrm.go
@@ -2,10 +2,11 @@ package centralstationcrm
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/centralstationcrm/centralstationcrm_integration_test.go b/pkg/detectors/centralstationcrm/centralstationcrm_integration_test.go
new file mode 100644
index 000000000..364ed8e82
--- /dev/null
+++ b/pkg/detectors/centralstationcrm/centralstationcrm_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package centralstationcrm
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCentralStationCRM_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CENTRALSTATIONCRM")
+ inactiveSecret := testSecrets.MustGetField("CENTRALSTATIONCRM_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CentralStationCRM,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CentralStationCRM,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CentralStationCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CentralStationCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/centralstationcrm/centralstationcrm_test.go b/pkg/detectors/centralstationcrm/centralstationcrm_test.go
index 364ed8e82..01fe54b65 100644
--- a/pkg/detectors/centralstationcrm/centralstationcrm_test.go
+++ b/pkg/detectors/centralstationcrm/centralstationcrm_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package centralstationcrm
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ centralstationcrm_api_key: "gyeyy7soy4lxx7yenw56iba4szfu1f"
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "gyeyy7soy4lxx7yenw56iba4szfu1f"
)
-func TestCentralStationCRM_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CENTRALSTATIONCRM")
- inactiveSecret := testSecrets.MustGetField("CENTRALSTATIONCRM_INACTIVE")
+func TestCentralStationCRM_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CentralStationCRM,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CentralStationCRM,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CentralStationCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CentralStationCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cexio/cexio.go b/pkg/detectors/cexio/cexio.go
index 56b06c020..3311240a4 100644
--- a/pkg/detectors/cexio/cexio.go
+++ b/pkg/detectors/cexio/cexio.go
@@ -6,7 +6,6 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"net/url"
@@ -14,12 +13,14 @@ import (
"strings"
"time"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/cexio/cexio_integration_test.go b/pkg/detectors/cexio/cexio_integration_test.go
new file mode 100644
index 000000000..0fd5b1422
--- /dev/null
+++ b/pkg/detectors/cexio/cexio_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cexio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCexIO_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ userId := testSecrets.MustGetField("CEXIO_USERID")
+ key := testSecrets.MustGetField("CEXIO_KEY")
+ inactiveKey := testSecrets.MustGetField("CEXIO_KEY_INACTIVE")
+ secret := testSecrets.MustGetField("CEXIO")
+ inactiveSecret := testSecrets.MustGetField("CEXIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within", userId, key, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CexIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within but not valid", userId, inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CexIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CexIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CexIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cexio/cexio_test.go b/pkg/detectors/cexio/cexio_test.go
index 0fd5b1422..36e8ced85 100644
--- a/pkg/detectors/cexio/cexio_test.go
+++ b/pkg/detectors/cexio/cexio_test.go
@@ -1,119 +1,98 @@
-//go:build detectors
-// +build detectors
-
package cexio
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCexIO_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- userId := testSecrets.MustGetField("CEXIO_USERID")
- key := testSecrets.MustGetField("CEXIO_KEY")
- inactiveKey := testSecrets.MustGetField("CEXIO_KEY_INACTIVE")
- secret := testSecrets.MustGetField("CEXIO")
- inactiveSecret := testSecrets.MustGetField("CEXIO_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ cexio_key: "bVI24QF2B8omVr9ddfxSHtkb18D"
+ cexio_secret: "2m2pEr2OLi48y2NCpSbPVwJqb"
+ cex.io_userID: "zd930167221"
+ auth_type: Signature
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secrets = []string{
+ "zd9301672212m2pEr2OLi48y2NCpSbPVwJqb",
+ "zd9301672212m2pEr2OLi48y2NCpSbPVwJqb",
+ "zd930167221bVI24QF2B8omVr9ddfxSHtkb18D",
+ "zd930167221bVI24QF2B8omVr9ddfxSHtkb18D",
}
+)
+
+func TestCexio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within", userId, key, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CexIO,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within but not valid", userId, inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CexIO,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: secrets,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CexIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CexIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/chartmogul/chartmogul.go b/pkg/detectors/chartmogul/chartmogul.go
index 5af07ec49..689a54b33 100644
--- a/pkg/detectors/chartmogul/chartmogul.go
+++ b/pkg/detectors/chartmogul/chartmogul.go
@@ -4,10 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/chartmogul/chartmogul_integration_test.go b/pkg/detectors/chartmogul/chartmogul_integration_test.go
new file mode 100644
index 000000000..31d43fb47
--- /dev/null
+++ b/pkg/detectors/chartmogul/chartmogul_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package chartmogul
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestChartmogul_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CHARTMOGUL")
+ inactiveSecret := testSecrets.MustGetField("CHARTMOGUL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Chartmogul,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Chartmogul,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Chartmogul.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Chartmogul.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/chartmogul/chartmogul_test.go b/pkg/detectors/chartmogul/chartmogul_test.go
index 31d43fb47..ac16af0b8 100644
--- a/pkg/detectors/chartmogul/chartmogul_test.go
+++ b/pkg/detectors/chartmogul/chartmogul_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package chartmogul
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ chartmogul_key: "e8hwspf91879g0u267yq1bkoxquvwndk"
+ auth_type: Basic
+ base_url: "https://api.example.com/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "e8hwspf91879g0u267yq1bkoxquvwndk"
)
-func TestChartmogul_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CHARTMOGUL")
- inactiveSecret := testSecrets.MustGetField("CHARTMOGUL_INACTIVE")
+func TestChartMogul_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Chartmogul,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Chartmogul,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Chartmogul.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Chartmogul.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/chatbot/chatbot_integration_test.go b/pkg/detectors/chatbot/chatbot_integration_test.go
new file mode 100644
index 000000000..ff62f3997
--- /dev/null
+++ b/pkg/detectors/chatbot/chatbot_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package chatbot
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestChatbot_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CHATBOT_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CHATBOT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a chatbot secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Chatbot,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a chatbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Chatbot,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Chatbot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Chatbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/chatbot/chatbot_test.go b/pkg/detectors/chatbot/chatbot_test.go
index ff62f3997..e7f3fff44 100644
--- a/pkg/detectors/chatbot/chatbot_test.go
+++ b/pkg/detectors/chatbot/chatbot_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package chatbot
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: Bearer
+ base_url: "https://api.example.com/v1/user"
+ chatbot_auth_token: "Bearer 5RzDGrpFKkrA_90yM_BFmyxKKAQkgu0B"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "5RzDGrpFKkrA_90yM_BFmyxKKAQkgu0B"
)
-func TestChatbot_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CHATBOT_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CHATBOT_INACTIVE")
+func TestChatBot_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a chatbot secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Chatbot,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a chatbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Chatbot,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Chatbot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Chatbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/chatfule/chatfule.go b/pkg/detectors/chatfule/chatfule.go
index d947ed7b7..067ca140a 100644
--- a/pkg/detectors/chatfule/chatfule.go
+++ b/pkg/detectors/chatfule/chatfule.go
@@ -3,10 +3,11 @@ package chatfule
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/chatfule/chatfule_integration_test.go b/pkg/detectors/chatfule/chatfule_integration_test.go
new file mode 100644
index 000000000..405174ed6
--- /dev/null
+++ b/pkg/detectors/chatfule/chatfule_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package chatfule
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestChatfule_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CHATFULE")
+ inactiveSecret := testSecrets.MustGetField("CHATFULE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Chatfule,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Chatfule,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Chatfuel.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Chatfuel.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/chatfule/chatfule_test.go b/pkg/detectors/chatfule/chatfule_test.go
index 405174ed6..0f98714ca 100644
--- a/pkg/detectors/chatfule/chatfule_test.go
+++ b/pkg/detectors/chatfule/chatfule_test.go
@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
package chatfule
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestChatfule_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CHATFULE")
- inactiveSecret := testSecrets.MustGetField("CHATFULE_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: Bearer
+ base_url: "https://api.example.com/v1/user"
+ chatfuel_auth_token: "Bearer 22ZEyoBNMXpT6rHmbuBlIJ8n19vo3tHzNTQDUku00WhHuBCAlkfkn8kQkXslseKEHARZthTrm8QfErQ5auXEr8teFIt6stHYi9sfJXM7IK0vEsezKFQwADCvMhX202eL"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "22ZEyoBNMXpT6rHmbuBlIJ8n19vo3tHzNTQDUku00WhHuBCAlkfkn8kQkXslseKEHARZthTrm8QfErQ5auXEr8teFIt6stHYi9sfJXM7IK0vEsezKFQwADCvMhX202eL"
+)
+
+func TestChatFule_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Chatfule,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Chatfule,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Chatfuel.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Chatfuel.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/checio/checio.go b/pkg/detectors/checio/checio.go
index c7a58d91d..5563c70b2 100644
--- a/pkg/detectors/checio/checio.go
+++ b/pkg/detectors/checio/checio.go
@@ -2,11 +2,11 @@ package checio
import (
"context"
- // "log"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/checio/checio_integration_test.go b/pkg/detectors/checio/checio_integration_test.go
new file mode 100644
index 000000000..e2fd6eafa
--- /dev/null
+++ b/pkg/detectors/checio/checio_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package checio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestChecIO_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CHECIO")
+ inactiveSecret := testSecrets.MustGetField("CHECIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a checio secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ChecIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a checio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ChecIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ChecIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ChecIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/checio/checio_test.go b/pkg/detectors/checio/checio_test.go
index e2fd6eafa..ae7a224bf 100644
--- a/pkg/detectors/checio/checio_test.go
+++ b/pkg/detectors/checio/checio_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package checio
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: N/A
+ base_url: "https://api.example.com/v1/user"
+ checio_auth_token: "X-Authorization pk_k64v4e7f5vfun5efk7kscnvuwiuo9ioxbvxjq8qrdga0p"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "pk_k64v4e7f5vfun5efk7kscnvuwiuo9ioxbvxjq8qrdga0p"
)
-func TestChecIO_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CHECIO")
- inactiveSecret := testSecrets.MustGetField("CHECIO_INACTIVE")
+func TestChecio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a checio secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ChecIO,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a checio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ChecIO,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ChecIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ChecIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/checklyhq/checklyhq.go b/pkg/detectors/checklyhq/checklyhq.go
index 274574b1c..607a331d2 100644
--- a/pkg/detectors/checklyhq/checklyhq.go
+++ b/pkg/detectors/checklyhq/checklyhq.go
@@ -3,10 +3,11 @@ package checklyhq
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/checklyhq/checklyhq_integration_test.go b/pkg/detectors/checklyhq/checklyhq_integration_test.go
new file mode 100644
index 000000000..8985924f7
--- /dev/null
+++ b/pkg/detectors/checklyhq/checklyhq_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package checklyhq
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestChecklyHQ_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CHECKLYHQ")
+ inactiveSecret := testSecrets.MustGetField("CHECKLYHQ_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ChecklyHQ,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ChecklyHQ,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ChecklyHQ.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ChecklyHQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/checklyhq/checklyhq_test.go b/pkg/detectors/checklyhq/checklyhq_test.go
index 8985924f7..ff1c031a0 100644
--- a/pkg/detectors/checklyhq/checklyhq_test.go
+++ b/pkg/detectors/checklyhq/checklyhq_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package checklyhq
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: Bearer
+ base_url: "https://api.example.com/v1/user"
+ checklyhq_auth_token: "Bearer r3sd5apfe7p3eg1318qpbtxo36gwcct2"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "r3sd5apfe7p3eg1318qpbtxo36gwcct2"
)
-func TestChecklyHQ_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CHECKLYHQ")
- inactiveSecret := testSecrets.MustGetField("CHECKLYHQ_INACTIVE")
+func TestChecklyhq_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ChecklyHQ,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ChecklyHQ,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ChecklyHQ.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ChecklyHQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/checkout/checkout.go b/pkg/detectors/checkout/checkout.go
index c14dcb129..def4425b7 100644
--- a/pkg/detectors/checkout/checkout.go
+++ b/pkg/detectors/checkout/checkout.go
@@ -2,10 +2,11 @@ package checkout
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/checkout/checkout_integration_test.go b/pkg/detectors/checkout/checkout_integration_test.go
new file mode 100644
index 000000000..0337cfe5b
--- /dev/null
+++ b/pkg/detectors/checkout/checkout_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package checkout
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCheckout_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CHECKOUT")
+ inactiveSecret := testSecrets.MustGetField("CHECKOUT_INACTIVE")
+ secretId := testSecrets.MustGetField("CHECKOUT_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within", secret, secretId)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Checkout,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within but not valid", inactiveSecret, secretId)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Checkout,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Checkout.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Checkout.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/checkout/checkout_test.go b/pkg/detectors/checkout/checkout_test.go
index 0337cfe5b..7132cb6f2 100644
--- a/pkg/detectors/checkout/checkout_test.go
+++ b/pkg/detectors/checkout/checkout_test.go
@@ -1,121 +1,91 @@
-//go:build detectors
-// +build detectors
-
package checkout
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: API-Key
+ base_url: "https://api.checkout.com/v1/customers/cus_DaZoK0ioakAfFaj6fyqSFQZatk"
+ checkout_api_key: "sk_test_14a67eEd-21Fd-B18d-2B8D-275697febE7D"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "sk_test_14a67eEd-21Fd-B18d-2B8D-275697febE7Dcus_DaZoK0ioakAfFaj6fyqSFQZatk"
)
-func TestCheckout_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CHECKOUT")
- inactiveSecret := testSecrets.MustGetField("CHECKOUT_INACTIVE")
- secretId := testSecrets.MustGetField("CHECKOUT_ID")
+func TestCheckout_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within", secret, secretId)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Checkout,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within but not valid", inactiveSecret, secretId)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Checkout,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Checkout.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Checkout.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cicero/cicero.go b/pkg/detectors/cicero/cicero.go
index 6014f49a8..3d64ae941 100644
--- a/pkg/detectors/cicero/cicero.go
+++ b/pkg/detectors/cicero/cicero.go
@@ -3,10 +3,11 @@ package cicero
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cicero/cicero_integration_test.go b/pkg/detectors/cicero/cicero_integration_test.go
new file mode 100644
index 000000000..7bb562b69
--- /dev/null
+++ b/pkg/detectors/cicero/cicero_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cicero
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCicero_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CICERO")
+ inactiveSecret := testSecrets.MustGetField("CICERO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cicero secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cicero,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cicero secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cicero,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cicero.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cicero.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cicero/cicero_test.go b/pkg/detectors/cicero/cicero_test.go
index 7bb562b69..617c7fa3c 100644
--- a/pkg/detectors/cicero/cicero_test.go
+++ b/pkg/detectors/cicero/cicero_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cicero
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ base_url: "https://api.cicero.com/v1/user?key=sbxod2yo56quitbyujhkig3mgtu6z49f4hh56va6"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "sbxod2yo56quitbyujhkig3mgtu6z49f4hh56va6"
)
-func TestCicero_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CICERO")
- inactiveSecret := testSecrets.MustGetField("CICERO_INACTIVE")
+func TestCicero_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cicero secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cicero,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cicero secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cicero,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cicero.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cicero.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/circleci/circleci.go b/pkg/detectors/circleci/circleci.go
index 6c04002ac..b81745bdf 100644
--- a/pkg/detectors/circleci/circleci.go
+++ b/pkg/detectors/circleci/circleci.go
@@ -2,9 +2,10 @@ package circleci
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/circleci/circleci_integration_test.go b/pkg/detectors/circleci/circleci_integration_test.go
new file mode 100644
index 000000000..34d685903
--- /dev/null
+++ b/pkg/detectors/circleci/circleci_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package circleci
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCircleCI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CIRCLECI")
+ secretInactive := testSecrets.MustGetField("CIRCLECI_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Circle,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Circle,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CircleCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CircleCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/circleci/circleci_test.go b/pkg/detectors/circleci/circleci_test.go
index 34d685903..1730c18f5 100644
--- a/pkg/detectors/circleci/circleci_test.go
+++ b/pkg/detectors/circleci/circleci_test.go
@@ -1,122 +1,91 @@
-//go:build detectors
-// +build detectors
-
package circleci
import (
"context"
- "fmt"
- "os"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCircleCI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CIRCLECI")
- secretInactive := testSecrets.MustGetField("CIRCLECI_INACTIVE")
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: API-Key
+ base_url: "https://api.example.com/v1/user"
+ circleci_api_key: "4a4fFEA0Cb7760ee42Bb1Dc82b1E4E5eDCacB9E7"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "4a4fFEA0Cb7760ee42Bb1Dc82b1E4E5eDCacB9E7"
+)
+
+func TestCircleCI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a circle secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Circle,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a circle secret %s within", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Circle,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CircleCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- if os.Getenv("FORCE_PASS_DIFF") == "true" {
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CircleCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clarifai/clarifai.go b/pkg/detectors/clarifai/clarifai.go
index 29018e406..3dce5439c 100644
--- a/pkg/detectors/clarifai/clarifai.go
+++ b/pkg/detectors/clarifai/clarifai.go
@@ -3,10 +3,11 @@ package clarifai
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clarifai/clarifai_integration_test.go b/pkg/detectors/clarifai/clarifai_integration_test.go
new file mode 100644
index 000000000..96ab6fe5a
--- /dev/null
+++ b/pkg/detectors/clarifai/clarifai_integration_test.go
@@ -0,0 +1,137 @@
+//go:build detectors
+// +build detectors
+
+package clarifai
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClarifai_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ apiKey := testSecrets.MustGetField("CLARIFAI_API_KEY")
+ inactiveApiKey := testSecrets.MustGetField("CLARIFAI_API_KEY_INACTIVE")
+ pat := testSecrets.MustGetField("CLARIFAI_PAT")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clarifai api key %s within", apiKey)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clarifai,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clarifai pat %s within", pat)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clarifai,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clarifai secret %s within but unverified", inactiveApiKey)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clarifai,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Clarifai.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Clarifai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clarifai/clarifai_test.go b/pkg/detectors/clarifai/clarifai_test.go
index 96ab6fe5a..62d226964 100644
--- a/pkg/detectors/clarifai/clarifai_test.go
+++ b/pkg/detectors/clarifai/clarifai_test.go
@@ -1,137 +1,91 @@
-//go:build detectors
-// +build detectors
-
package clarifai
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ api:
+ auth_type: API-Key
+ base_url: "https://api.example.com/v1/user"
+ clarifai_key: "WCFcfUCsl2P3vCJuFgDuLeUXduycoZli"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "WCFcfUCsl2P3vCJuFgDuLeUXduycoZli"
)
-func TestClarifai_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- apiKey := testSecrets.MustGetField("CLARIFAI_API_KEY")
- inactiveApiKey := testSecrets.MustGetField("CLARIFAI_API_KEY_INACTIVE")
- pat := testSecrets.MustGetField("CLARIFAI_PAT")
+func TestClarifai_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clarifai api key %s within", apiKey)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clarifai,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clarifai pat %s within", pat)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clarifai,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clarifai secret %s within but unverified", inactiveApiKey)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clarifai,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Clarifai.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Clarifai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clearbit/clearbit.go b/pkg/detectors/clearbit/clearbit.go
index ef56a93ab..b45a2b28a 100644
--- a/pkg/detectors/clearbit/clearbit.go
+++ b/pkg/detectors/clearbit/clearbit.go
@@ -3,10 +3,11 @@ package clearbit
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clearbit/clearbit_integration_test.go b/pkg/detectors/clearbit/clearbit_integration_test.go
new file mode 100644
index 000000000..da8ba1c26
--- /dev/null
+++ b/pkg/detectors/clearbit/clearbit_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package clearbit
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClearbit_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLEARBIT")
+ inactiveSecret := testSecrets.MustGetField("CLEARBIT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clearbit secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clearbit,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clearbit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clearbit,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Clearbit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Clearbit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clearbit/clearbit_test.go b/pkg/detectors/clearbit/clearbit_test.go
index da8ba1c26..522d69119 100644
--- a/pkg/detectors/clearbit/clearbit_test.go
+++ b/pkg/detectors/clearbit/clearbit_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package clearbit
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: Bearer
+ base_url: "https://api.example.com/v1/user"
+ clearbit_auth_token: "Bearer 9itvicgfiyq3ry6g03qwhwc_0s309dy8woh"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "9itvicgfiyq3ry6g03qwhwc_0s309dy8woh"
)
-func TestClearbit_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLEARBIT")
- inactiveSecret := testSecrets.MustGetField("CLEARBIT_INACTIVE")
+func TestClearBit_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clearbit secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clearbit,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clearbit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clearbit,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Clearbit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Clearbit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clickhelp/clickhelp.go b/pkg/detectors/clickhelp/clickhelp.go
index 3d2f839f6..f258f8dd8 100644
--- a/pkg/detectors/clickhelp/clickhelp.go
+++ b/pkg/detectors/clickhelp/clickhelp.go
@@ -4,16 +4,17 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/clickhelp/clickhelp_integration_test.go b/pkg/detectors/clickhelp/clickhelp_integration_test.go
new file mode 100644
index 000000000..16965f63d
--- /dev/null
+++ b/pkg/detectors/clickhelp/clickhelp_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package clickhelp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClickhelp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ email := testSecrets.MustGetField("SCANNERS_EMAIL")
+ server := testSecrets.MustGetField("CLICKHELP_SERVER")
+ key := testSecrets.MustGetField("CLICKHELP")
+ inactiveKey := testSecrets.MustGetField("CLICKHELP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s", key, email, server)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClickHelp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s but not valid", inactiveKey, email, server)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClickHelp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Clickhelp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Clickhelp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clickhelp/clickhelp_test.go b/pkg/detectors/clickhelp/clickhelp_test.go
index 16965f63d..f8ce474a0 100644
--- a/pkg/detectors/clickhelp/clickhelp_test.go
+++ b/pkg/detectors/clickhelp/clickhelp_test.go
@@ -1,122 +1,97 @@
-//go:build detectors
-// +build detectors
-
package clickhelp
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestClickhelp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- email := testSecrets.MustGetField("SCANNERS_EMAIL")
- server := testSecrets.MustGetField("CLICKHELP_SERVER")
- key := testSecrets.MustGetField("CLICKHELP")
- inactiveKey := testSecrets.MustGetField("CLICKHELP_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ api:
+ user_email: "[email protected]"
+ clickhelp_key: "XzUkp562BtmjfRGoOGBiLLNu"
+ clickhelp_domain: testingdev.try.clickhelp.co
+ auth_type: Basic
+ base_url: "https://testing-dev.try.clickhelp.co/v1/user"
+ auth_token: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secrets = []string{
}
+)
+
+func TestClickHelp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s", key, email, server)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClickHelp,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s but not valid", inactiveKey, email, server)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClickHelp,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: secrets,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Clickhelp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Clickhelp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clicksendsms/clicksendsms.go b/pkg/detectors/clicksendsms/clicksendsms.go
index f036f39e4..67bd60fed 100644
--- a/pkg/detectors/clicksendsms/clicksendsms.go
+++ b/pkg/detectors/clicksendsms/clicksendsms.go
@@ -4,16 +4,17 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/clicksendsms/clicksendsms_integration_test.go b/pkg/detectors/clicksendsms/clicksendsms_integration_test.go
new file mode 100644
index 000000000..9df9c4c63
--- /dev/null
+++ b/pkg/detectors/clicksendsms/clicksendsms_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package clicksendsms
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClickSendsms_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLICKSENDSMS_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CLICKSENDSMS_INACTIVE")
+ email := testSecrets.MustGetField("CLICKSENDSMS_EMAIL")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s", secret, email)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClickSendsms,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s but not valid", inactiveSecret, email)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClickSendsms,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ClickSendsms.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ClickSendsms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clicksendsms/clicksendsms_test.go b/pkg/detectors/clicksendsms/clicksendsms_test.go
index 9df9c4c63..4bdc3dbca 100644
--- a/pkg/detectors/clicksendsms/clicksendsms_test.go
+++ b/pkg/detectors/clicksendsms/clicksendsms_test.go
@@ -1,121 +1,92 @@
-//go:build detectors
-// +build detectors
-
package clicksendsms
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File for clicksendsms: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: Basic
+ base_url: "https://api.example.com/v1/sms"
+ sms_id: G9TXU2YD-NOYB-LLSX-21NU-5CX5SIA330Z7
+ sms_email: [email protected]
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "[email protected]"
)
-func TestClickSendsms_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLICKSENDSMS_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CLICKSENDSMS_INACTIVE")
- email := testSecrets.MustGetField("CLICKSENDSMS_EMAIL")
+func TestClickSendSMS_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s", secret, email)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClickSendsms,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s but not valid", inactiveSecret, email)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClickSendsms,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ClickSendsms.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ClickSendsms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go
index 45d713562..af57edeba 100644
--- a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go
+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go
@@ -2,10 +2,11 @@ package clickuppersonaltoken
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_integration_test.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_integration_test.go
new file mode 100644
index 000000000..8c8bd60f7
--- /dev/null
+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package clickuppersonaltoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClickupPersonalToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN")
+ inactiveSecret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClickupPersonalToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClickupPersonalToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ClickupPersonalToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ClickupPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go
index 8c8bd60f7..673c0921a 100644
--- a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go
+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package clickuppersonaltoken
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ base_url: "https://api.example.com/v1/user"
+ clickup_token: "pk_7043602_WIKY22PAKCVC1S5Q2X6119IK7N1UL8VY"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "pk_7043602_WIKY22PAKCVC1S5Q2X6119IK7N1UL8VY"
)
-func TestClickupPersonalToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN")
- inactiveSecret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN_INACTIVE")
+func TestClickupPersonalToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClickupPersonalToken,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClickupPersonalToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ClickupPersonalToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ClickupPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cliengo/cliengo.go b/pkg/detectors/cliengo/cliengo.go
index 54dcfbad8..3615eef20 100644
--- a/pkg/detectors/cliengo/cliengo.go
+++ b/pkg/detectors/cliengo/cliengo.go
@@ -2,10 +2,11 @@ package cliengo
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cliengo/cliengo_integration_test.go b/pkg/detectors/cliengo/cliengo_integration_test.go
new file mode 100644
index 000000000..1d31d5cf0
--- /dev/null
+++ b/pkg/detectors/cliengo/cliengo_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package cliengo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCliengo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLIENGO")
+ inactiveSecret := testSecrets.MustGetField("CLIENGO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cliengo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cliengo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cliengo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cliengo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cliengo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cliengo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cliengo/cliengo_test.go b/pkg/detectors/cliengo/cliengo_test.go
index 1d31d5cf0..e6250b874 100644
--- a/pkg/detectors/cliengo/cliengo_test.go
+++ b/pkg/detectors/cliengo/cliengo_test.go
@@ -1,116 +1,89 @@
-//go:build detectors
-// +build detectors
-
package cliengo
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCliengo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLIENGO")
- inactiveSecret := testSecrets.MustGetField("CLIENGO_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.cliengo.com/v1/user?key=9e4635bc-28dc-25d3-8546-2b30115d9a7b"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "9e4635bc-28dc-25d3-8546-2b30115d9a7b"
+)
+
+func TestCliengo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cliengo secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cliengo,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cliengo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cliengo,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cliengo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cliengo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/clinchpad/clinchpad.go b/pkg/detectors/clinchpad/clinchpad.go
index 09357e0fe..c39d9727f 100644
--- a/pkg/detectors/clinchpad/clinchpad.go
+++ b/pkg/detectors/clinchpad/clinchpad.go
@@ -4,10 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clinchpad/clinchpad_integration_test.go b/pkg/detectors/clinchpad/clinchpad_integration_test.go
new file mode 100644
index 000000000..68d05a39f
--- /dev/null
+++ b/pkg/detectors/clinchpad/clinchpad_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package clinchpad
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClinchpad_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLINCHPAD_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CLINCHPAD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clinchpad,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clinchpad,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Clinchpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Clinchpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clinchpad/clinchpad_test.go b/pkg/detectors/clinchpad/clinchpad_test.go
index 68d05a39f..651f1d960 100644
--- a/pkg/detectors/clinchpad/clinchpad_test.go
+++ b/pkg/detectors/clinchpad/clinchpad_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package clinchpad
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Basic"
+ base_url: "https://api.example.com/v1/user"
+ clinchpad_key: "3v1xo5r03ghc538iwzbzeddwqulnun8h"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "3v1xo5r03ghc538iwzbzeddwqulnun8h"
)
-func TestClinchpad_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLINCHPAD_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CLINCHPAD_INACTIVE")
+func TestClinchPad_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clinchpad,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clinchpad,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Clinchpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Clinchpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clockify/clockify.go b/pkg/detectors/clockify/clockify.go
index e9414f4a7..f89768242 100644
--- a/pkg/detectors/clockify/clockify.go
+++ b/pkg/detectors/clockify/clockify.go
@@ -2,10 +2,11 @@ package clockify
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clockify/clockify_integration_test.go b/pkg/detectors/clockify/clockify_integration_test.go
new file mode 100644
index 000000000..618512566
--- /dev/null
+++ b/pkg/detectors/clockify/clockify_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package clockify
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClockify_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOCKIFY")
+ inactiveSecret := testSecrets.MustGetField("CLOCKIFY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clockify secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clockify,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clockify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Clockify,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Clockify.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Clockify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clockify/clockify_test.go b/pkg/detectors/clockify/clockify_test.go
index 618512566..cff7b09b2 100644
--- a/pkg/detectors/clockify/clockify_test.go
+++ b/pkg/detectors/clockify/clockify_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package clockify
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.example.com/v1/user"
+ clockify_key: "kfJkRn7Knahh6pyDOL82NqNq5c4VLUNulVe5CMyJpIK9NXQC"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "kfJkRn7Knahh6pyDOL82NqNq5c4VLUNulVe5CMyJpIK9NXQC"
)
-func TestClockify_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOCKIFY")
- inactiveSecret := testSecrets.MustGetField("CLOCKIFY_INACTIVE")
+func TestClockify_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clockify secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clockify,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clockify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Clockify,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Clockify.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Clockify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clockworksms/clockworksms.go b/pkg/detectors/clockworksms/clockworksms.go
index 1b5904893..3924adcc7 100644
--- a/pkg/detectors/clockworksms/clockworksms.go
+++ b/pkg/detectors/clockworksms/clockworksms.go
@@ -2,10 +2,11 @@ package clockworksms
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clockworksms/clockworksms_integration_test.go b/pkg/detectors/clockworksms/clockworksms_integration_test.go
new file mode 100644
index 000000000..4250011b2
--- /dev/null
+++ b/pkg/detectors/clockworksms/clockworksms_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package clockworksms
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClockworksms_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ userKey := testSecrets.MustGetField("CLOCKWORKSMS_USERKEY")
+ token := testSecrets.MustGetField("CLOCKWORKSMS_TOKEN")
+ inactiveToken := testSecrets.MustGetField("CLOCKWORKSMS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within", userKey, token)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClockworkSMS,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within but not valid", userKey, inactiveToken)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClockworkSMS,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Clockworksms.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Clockworksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clockworksms/clockworksms_test.go b/pkg/detectors/clockworksms/clockworksms_test.go
index 4250011b2..40f0c3981 100644
--- a/pkg/detectors/clockworksms/clockworksms_test.go
+++ b/pkg/detectors/clockworksms/clockworksms_test.go
@@ -1,121 +1,92 @@
-//go:build detectors
-// +build detectors
-
package clockworksms
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ base_url: "https://api.textanywhere.com/v1/user"
+ clockwork_key: "84473"
+ clockwork_token: "YROh7NZbZxHwiSc9pMNIAGYs"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "84473YROh7NZbZxHwiSc9pMNIAGYs"
)
-func TestClockworksms_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- userKey := testSecrets.MustGetField("CLOCKWORKSMS_USERKEY")
- token := testSecrets.MustGetField("CLOCKWORKSMS_TOKEN")
- inactiveToken := testSecrets.MustGetField("CLOCKWORKSMS_INACTIVE")
+func TestClockWorkSMS_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within", userKey, token)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClockworkSMS,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within but not valid", userKey, inactiveToken)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClockworkSMS,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Clockworksms.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Clockworksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/closecrm/close.go b/pkg/detectors/closecrm/close.go
index de0cf4967..00db62924 100644
--- a/pkg/detectors/closecrm/close.go
+++ b/pkg/detectors/closecrm/close.go
@@ -4,10 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/closecrm/close_integration_test.go b/pkg/detectors/closecrm/close_integration_test.go
new file mode 100644
index 000000000..2a4940dc4
--- /dev/null
+++ b/pkg/detectors/closecrm/close_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package closecrm
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClose_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOSE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CLOSE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a close secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Close,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a close secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Close,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Close.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Close.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/closecrm/close_test.go b/pkg/detectors/closecrm/close_test.go
index 2a4940dc4..e2a2bb59a 100644
--- a/pkg/detectors/closecrm/close_test.go
+++ b/pkg/detectors/closecrm/close_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package closecrm
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Basic"
+ base_url: "https://api.example.com/v1/user"
+ close_key: "api_3cyEW8syFEmeND561qJ9Sj8mT6E0VyWqY7h6cjJIBtc2e"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "api_3cyEW8syFEmeND561qJ9Sj8mT6E0VyWqY7h6cjJIBtc2e"
)
-func TestClose_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOSE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CLOSE_INACTIVE")
+func TestCloseCRM_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a close secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Close,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a close secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Close,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Close.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Close.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudconvert/cloudconvert_integration_test.go b/pkg/detectors/cloudconvert/cloudconvert_integration_test.go
new file mode 100644
index 000000000..9109b4a58
--- /dev/null
+++ b/pkg/detectors/cloudconvert/cloudconvert_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cloudconvert
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudConvert_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDCONVERT")
+ inactiveSecret := testSecrets.MustGetField("CLOUDCONVERT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudConvert,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudConvert,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CloudConvert.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CloudConvert.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudconvert/cloudconvert_test.go b/pkg/detectors/cloudconvert/cloudconvert_test.go
index 9109b4a58..5caac3f9e 100644
--- a/pkg/detectors/cloudconvert/cloudconvert_test.go
+++ b/pkg/detectors/cloudconvert/cloudconvert_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloudconvert
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ cloudconvert_key: "eypn1gEV3BnckI3jcYzUvliSbukvxzO5acvE?ey8VEaR0lmpRa4IXv02fDPnlfdukWtb1/p-nlQlPVGnB52f9KwY4q98aVghXZqoit4AeFxMAHcCytOj61o8lHdcUF9fIcyF2HaFIk/k3Hdt7pS/5rb2eeWcEvc-5XB0T_Oh68AtCG8mOPpwKvzrhhIuEJck3vtFncgbDrSxg5mKkw924rMLP3Tb5tgIRuawZLwBxJL/qVIhAzfDGiIeNTzYOB9zHfHlfw3aZ1i/terePSN5EafVJ1yYw1KRXWL9/kPdAO0yFwSv3mUWx04oIIUURG6QKwO0rk7L0eAxnu4djnSXtqdvH_G50H1SSwwfKUg2Xz25-OZLkhxiaxEMMMY=3x0Yjhs7O1KFkI5gUQKH_VYAU2bJSAqpCKsxaYrdw91wUoya5rflCBVDHjC/BsezIkPFFmEu7sqs3WJg6dZeAiguYx7uZtDx1ILH18f29q9o34bM9SNolZNcG3fN7L2eWjilbmUq/Ty2545WkbHTjlcjLlHPAAjzLebfcFnlMSKH9Tqb/qx3G1z8wfzMa3dn3iRqNHwfmGOmfgK7RjtlZwoVruMjDWEza/o8imZF513yM7FrHTJkTFa1JjVbjU/C85ItZTiJsBUKAt/DbLg6W7lieKgHbgmz3cuwgVR7YDLZJB056TRcU9wrV0SUYDz0gogrpOEnZxdo4fb5UcCllj/AD/dYsfqVSHtTxKWBhun9Iqmx8FjgPtFCFugTxfaaHZ9dUC7TPahdSxixGvnu8EEvAs0Te85eJ9iyeq628Tvboz9J7KMq/uwflJtecSquJiWJT9GsYL5dl3Hr6ZYhxqs1-mrrB5FNzn-NPclPSu9PANtQ1BDuahKy683/t85F8yjug5C5paamNfgiJgOm5Vi/USUmWeVmH_htZoYGJTbOywDkRT1bYp9JIxlWHA29MInhWNrdlxZ_1h-SQ3fM6pzKIoJ0m_T/KXYERPzle0cy_/OnlfIa-yUgBnx_slQ1f9h0AS/PVMv/yZ6W"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "eypn1gEV3BnckI3jcYzUvliSbukvxzO5acvE?ey8VEaR0lmpRa4IXv02fDPnlfdukWtb1/p-nlQlPVGnB52f9KwY4q98aVghXZqoit4AeFxMAHcCytOj61o8lHdcUF9fIcyF2HaFIk/k3Hdt7pS/5rb2eeWcEvc-5XB0T_Oh68AtCG8mOPpwKvzrhhIuEJck3vtFncgbDrSxg5mKkw924rMLP3Tb5tgIRuawZLwBxJL/qVIhAzfDGiIeNTzYOB9zHfHlfw3aZ1i/terePSN5EafVJ1yYw1KRXWL9/kPdAO0yFwSv3mUWx04oIIUURG6QKwO0rk7L0eAxnu4djnSXtqdvH_G50H1SSwwfKUg2Xz25-OZLkhxiaxEMMMY=3x0Yjhs7O1KFkI5gUQKH_VYAU2bJSAqpCKsxaYrdw91wUoya5rflCBVDHjC/BsezIkPFFmEu7sqs3WJg6dZeAiguYx7uZtDx1ILH18f29q9o34bM9SNolZNcG3fN7L2eWjilbmUq/Ty2545WkbHTjlcjLlHPAAjzLebfcFnlMSKH9Tqb/qx3G1z8wfzMa3dn3iRqNHwfmGOmfgK7RjtlZwoVruMjDWEza/o8imZF513yM7FrHTJkTFa1JjVbjU/C85ItZTiJsBUKAt/DbLg6W7lieKgHbgmz3cuwgVR7YDLZJB056TRcU9wrV0SUYDz0gogrpOEnZxdo4fb5UcCllj/AD/dYsfqVSHtTxKWBhun9Iqmx8FjgPtFCFugTxfaaHZ9dUC7TPahdSxixGvnu8EEvAs0Te85eJ9iyeq628Tvboz9J7KMq/uwflJtecSquJiWJT9GsYL5dl3Hr6ZYhxqs1-mrrB5FNzn-NPclPSu9PANtQ1BDuahKy683/t85F8yjug5C5paamNfgiJgOm5Vi/USUmWeVmH_htZoYGJTbOywDkRT1bYp9JIxlWHA29MInhWNrdlxZ_1h-SQ3fM6pzKIoJ0m_T/KXYERPzle0cy_/OnlfIa-yUgBnx_slQ1f9h0AS/PVMv/yZ6W"
)
-func TestCloudConvert_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDCONVERT")
- inactiveSecret := testSecrets.MustGetField("CLOUDCONVERT_INACTIVE")
+func TestCloudConvert_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudConvert,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudConvert,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CloudConvert.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CloudConvert.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudelements/cloudelements.go b/pkg/detectors/cloudelements/cloudelements.go
index 9c77ab362..19002fcdb 100644
--- a/pkg/detectors/cloudelements/cloudelements.go
+++ b/pkg/detectors/cloudelements/cloudelements.go
@@ -3,16 +3,17 @@ package cloudelements
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/cloudelements/cloudelements_integration_test.go b/pkg/detectors/cloudelements/cloudelements_integration_test.go
new file mode 100644
index 000000000..6fa4314dd
--- /dev/null
+++ b/pkg/detectors/cloudelements/cloudelements_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package cloudelements
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudElements_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDELEMENTS")
+ org := testSecrets.MustGetField("CLOUDELEMENTS_ORG")
+ inactiveSecret := testSecrets.MustGetField("CLOUDELEMENTS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s", secret, org)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudElements,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudElements,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CloudElements.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CloudElements.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudelements/cloudelements_test.go b/pkg/detectors/cloudelements/cloudelements_test.go
index 6fa4314dd..9cda22b9e 100644
--- a/pkg/detectors/cloudelements/cloudelements_test.go
+++ b/pkg/detectors/cloudelements/cloudelements_test.go
@@ -1,121 +1,92 @@
-//go:build detectors
-// +build detectors
-
package cloudelements
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ base_url: "https://api.example.com/v1/user"
+ cloudelements_key: "4NloL5EzH3PLvNzjCMikofUfKXYOsYOeJBopEyDScIL"
+ cloudelements_org: "inz9qofvjwnx59hgefq9sy5v64ilqrnu"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "4NloL5EzH3PLvNzjCMikofUfKXYOsYOeJBopEyDScILinz9qofvjwnx59hgefq9sy5v64ilqrnu"
)
-func TestCloudElements_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDELEMENTS")
- org := testSecrets.MustGetField("CLOUDELEMENTS_ORG")
- inactiveSecret := testSecrets.MustGetField("CLOUDELEMENTS_INACTIVE")
+func TestCloudElements_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s", secret, org)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudElements,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudElements,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CloudElements.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CloudElements.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go
index 8fa867c6f..18a7bb610 100644
--- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go
+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go
@@ -3,10 +3,11 @@ package cloudflareapitoken
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go
new file mode 100644
index 000000000..05660e2c5
--- /dev/null
+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go
@@ -0,0 +1,137 @@
+//go:build detectors
+// +build detectors
+
+package cloudflareapitoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudflareApiToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_API_INACTIVE")
+ secret2 := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN2")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudflareApiToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret2)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudflareApiToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within but unverified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudflareApiToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CloudflareApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CloudflareApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go
index 05660e2c5..22417496c 100644
--- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go
+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go
@@ -1,137 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloudflareapitoken
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ cloudflare_token: "kOjD1yceduu2jxL2uuwT9dkOIudU3_54sLCEud6j"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "kOjD1yceduu2jxL2uuwT9dkOIudU3_54sLCEud6j"
)
-func TestCloudflareApiToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_API_INACTIVE")
- secret2 := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN2")
+func TestCloudFlareAPIToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudflareApiToken,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret2)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudflareApiToken,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within but unverified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudflareApiToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CloudflareApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CloudflareApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey.go b/pkg/detectors/cloudflarecakey/cloudflarecakey.go
index 69499d35e..d74c59d15 100644
--- a/pkg/detectors/cloudflarecakey/cloudflarecakey.go
+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey.go
@@ -5,10 +5,11 @@ import (
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
- regexp "github.com/wasilibs/go-re2"
)
type Scanner struct{}
diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey_integration_test.go b/pkg/detectors/cloudflarecakey/cloudflarecakey_integration_test.go
new file mode 100644
index 000000000..0eedf05b9
--- /dev/null
+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package cloudflarecakey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudflareCaKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY")
+ inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudflareCaKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within but unverified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudflareCaKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CloudflareCaKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CloudflareCaKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go
index 0eedf05b9..3e1a45ff2 100644
--- a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go
+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go
@@ -1,122 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloudflarecakey
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.example.com/v1/user"
+ cloudflare: "vR2hGdmAyzLxI4kR34v-fgtaIug7VC_xudRuku6UQd5t9RoiY3zGiuVxMGiN6qR6O0cGjbGepFsHHrfm-9QmBahS.tvHRjYvf_c5XOKReWeVXsp3T-_gilqReVlS-U_9vj1tcMAwRjWCcehKiCOPnXEeN.Xg.lmwp0.mAIsgbrdPJsJE4hz7OF5PQ7D"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "vR2hGdmAyzLxI4kR34v-fgtaIug7VC_xudRuku6UQd5t9RoiY3zGiuVxMGiN6qR6O0cGjbGepFsHHrfm-9QmBahS.tvHRjYvf_c5XOKReWeVXsp3T-_gilqReVlS-U_9vj1tcMAwRjWCcehKiCOPnXEeN.Xg.lmwp0.mAIsgbrdPJsJE4hz7OF5PQ7D"
)
-func TestCloudflareCaKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY")
- inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY_INACTIVE")
+func TestCloudFlareCAKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudflareCaKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
+ }
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within but unverified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudflareCaKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CloudflareCaKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CloudflareCaKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudimage/cloudimage.go b/pkg/detectors/cloudimage/cloudimage.go
index 51a02c0f1..06bd2905e 100644
--- a/pkg/detectors/cloudimage/cloudimage.go
+++ b/pkg/detectors/cloudimage/cloudimage.go
@@ -2,11 +2,12 @@ package cloudimage
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"time"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cloudimage/cloudimage_integration_test.go b/pkg/detectors/cloudimage/cloudimage_integration_test.go
new file mode 100644
index 000000000..00e00f173
--- /dev/null
+++ b/pkg/detectors/cloudimage/cloudimage_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package cloudimage
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudImage_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDIMAGE")
+ inactiveSecret := testSecrets.MustGetField("CLOUDIMAGE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudImage,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CloudImage,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CloudImage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CloudImage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudimage/cloudimage_test.go b/pkg/detectors/cloudimage/cloudimage_test.go
index 00e00f173..0c329a4ec 100644
--- a/pkg/detectors/cloudimage/cloudimage_test.go
+++ b/pkg/detectors/cloudimage/cloudimage_test.go
@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
package cloudimage
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCloudImage_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDIMAGE")
- inactiveSecret := testSecrets.MustGetField("CLOUDIMAGE_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ base_url: "https://api.example.com/v1/user"
+ cloudimage: "d__9rvli8sm4jo18v5q0q4n7vhkwbv"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "d__9rvli8sm4jo18v5q0q4n7vhkwbv"
+)
+
+func TestCloudImage_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudImage,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CloudImage,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CloudImage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CloudImage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/cloudmersive/cloudmersive.go b/pkg/detectors/cloudmersive/cloudmersive.go
index b4ef56fc1..36cc9ae58 100644
--- a/pkg/detectors/cloudmersive/cloudmersive.go
+++ b/pkg/detectors/cloudmersive/cloudmersive.go
@@ -2,10 +2,11 @@ package cloudmersive
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cloudmersive/cloudmersive_integration_test.go b/pkg/detectors/cloudmersive/cloudmersive_integration_test.go
new file mode 100644
index 000000000..e0acea511
--- /dev/null
+++ b/pkg/detectors/cloudmersive/cloudmersive_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cloudmersive
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudmersive_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDMERSIVE")
+ inactiveSecret := testSecrets.MustGetField("CLOUDMERSIVE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloudmersive,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloudmersive,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cloudmersive.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cloudmersive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudmersive/cloudmersive_test.go b/pkg/detectors/cloudmersive/cloudmersive_test.go
index e0acea511..c8187c1fb 100644
--- a/pkg/detectors/cloudmersive/cloudmersive_test.go
+++ b/pkg/detectors/cloudmersive/cloudmersive_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloudmersive
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.example.com/v1/user"
+ cloudmersive: "sxk5k1nfra8jak0mjjc6afr6v-6gsf7dr9o1"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "sxk5k1nfra8jak0mjjc6afr6v-6gsf7dr9o1"
)
-func TestCloudmersive_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDMERSIVE")
- inactiveSecret := testSecrets.MustGetField("CLOUDMERSIVE_INACTIVE")
+func TestCloudMersive_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloudmersive,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloudmersive,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cloudmersive.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cloudmersive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudplan/cloudplan_integration_test.go b/pkg/detectors/cloudplan/cloudplan_integration_test.go
new file mode 100644
index 000000000..531d959b4
--- /dev/null
+++ b/pkg/detectors/cloudplan/cloudplan_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cloudplan
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudplan_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDPLAN")
+ inactiveSecret := testSecrets.MustGetField("CLOUDPLAN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloudplan,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloudplan,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cloudplan.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cloudplan.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudplan/cloudplan_test.go b/pkg/detectors/cloudplan/cloudplan_test.go
index 531d959b4..ab2f5c9b5 100644
--- a/pkg/detectors/cloudplan/cloudplan_test.go
+++ b/pkg/detectors/cloudplan/cloudplan_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloudplan
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ base_url: "https://api.example.com/v1/user"
+ cloudplan_session_key: "Y6D1FIS3XZXIJLKD82P6U8IXYV4UEYPP"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "Y6D1FIS3XZXIJLKD82P6U8IXYV4UEYPP"
)
-func TestCloudplan_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDPLAN")
- inactiveSecret := testSecrets.MustGetField("CLOUDPLAN_INACTIVE")
+func TestCloudPlan_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloudplan,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloudplan,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cloudplan.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cloudplan.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloudsmith/cloudsmith_integration_test.go b/pkg/detectors/cloudsmith/cloudsmith_integration_test.go
new file mode 100644
index 000000000..58fe57574
--- /dev/null
+++ b/pkg/detectors/cloudsmith/cloudsmith_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cloudsmith
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloudsmith_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOUDSMITH")
+ inactiveSecret := testSecrets.MustGetField("CLOUDSMITH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloudsmith,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloudsmith,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cloudsmith.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cloudsmith.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloudsmith/cloudsmith_test.go b/pkg/detectors/cloudsmith/cloudsmith_test.go
index 58fe57574..86aa492dd 100644
--- a/pkg/detectors/cloudsmith/cloudsmith_test.go
+++ b/pkg/detectors/cloudsmith/cloudsmith_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloudsmith
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "X-API-Key"
+ base_url: "https://api.example.com/v1/user"
+ cloudsmith: "6fd00a2cfd7bbc51e1c4db6ac2f29d59629afd22"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "6fd00a2cfd7bbc51e1c4db6ac2f29d59629afd22"
)
-func TestCloudsmith_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOUDSMITH")
- inactiveSecret := testSecrets.MustGetField("CLOUDSMITH_INACTIVE")
+func TestCloudSmith_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloudsmith,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloudsmith,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cloudsmith.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cloudsmith.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cloverly/cloverly.go b/pkg/detectors/cloverly/cloverly.go
index 6cadfbb5e..3d17b1323 100644
--- a/pkg/detectors/cloverly/cloverly.go
+++ b/pkg/detectors/cloverly/cloverly.go
@@ -3,10 +3,11 @@ package cloverly
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cloverly/cloverly_integration_test.go b/pkg/detectors/cloverly/cloverly_integration_test.go
new file mode 100644
index 000000000..38d6b72a0
--- /dev/null
+++ b/pkg/detectors/cloverly/cloverly_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cloverly
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCloverly_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLOVERLY")
+ inactiveSecret := testSecrets.MustGetField("CLOVERLY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloverly secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloverly,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cloverly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Cloverly,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Cloverly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Cloverly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cloverly/cloverly_test.go b/pkg/detectors/cloverly/cloverly_test.go
index cb4d64794..c1929ed64 100644
--- a/pkg/detectors/cloverly/cloverly_test.go
+++ b/pkg/detectors/cloverly/cloverly_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package cloverly
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ cloverly_token: "564i_0a9_v58bn:p9st3r3cgi95_"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "564i_0a9_v58bn:p9st3r3cgi95_"
)
-func TestCloverly_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLOVERLY")
- inactiveSecret := testSecrets.MustGetField("CLOVERLY_INACTIVE")
+func TestCloverly_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloverly secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloverly,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cloverly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Cloverly,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Cloverly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Cloverly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/clustdoc/clustdoc.go b/pkg/detectors/clustdoc/clustdoc.go
index cd1f32866..14d70d695 100644
--- a/pkg/detectors/clustdoc/clustdoc.go
+++ b/pkg/detectors/clustdoc/clustdoc.go
@@ -3,10 +3,11 @@ package clustdoc
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/clustdoc/clustdoc_integration_test.go b/pkg/detectors/clustdoc/clustdoc_integration_test.go
new file mode 100644
index 000000000..35802efce
--- /dev/null
+++ b/pkg/detectors/clustdoc/clustdoc_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package clustdoc
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestClustDoc_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CLUSTDOC")
+ inactiveSecret := testSecrets.MustGetField("CLUSTDOC_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClustDoc,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ClustDoc,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ClustDoc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ClustDoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/clustdoc/clustdoc_test.go b/pkg/detectors/clustdoc/clustdoc_test.go
index 35802efce..d3f08c59a 100644
--- a/pkg/detectors/clustdoc/clustdoc_test.go
+++ b/pkg/detectors/clustdoc/clustdoc_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package clustdoc
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ clustdoc_token: "yQ7mTTO4eJ4I9GHDEdzF3wq0KVowNKjPMud3q0ZqaEIuoR1qCrARUyLwknNP"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "yQ7mTTO4eJ4I9GHDEdzF3wq0KVowNKjPMud3q0ZqaEIuoR1qCrARUyLwknNP"
)
-func TestClustDoc_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CLUSTDOC")
- inactiveSecret := testSecrets.MustGetField("CLUSTDOC_INACTIVE")
+func TestClustDoc_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClustDoc,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ClustDoc,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ClustDoc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ClustDoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/coda/coda.go b/pkg/detectors/coda/coda.go
index 74e0b6ce7..c4c32d72d 100644
--- a/pkg/detectors/coda/coda.go
+++ b/pkg/detectors/coda/coda.go
@@ -3,10 +3,11 @@ package coda
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/coda/coda_integration_test.go b/pkg/detectors/coda/coda_integration_test.go
new file mode 100644
index 000000000..4b38faab8
--- /dev/null
+++ b/pkg/detectors/coda/coda_integration_test.go
@@ -0,0 +1,161 @@
+//go:build detectors
+// +build detectors
+
+package coda
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoda_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CODA")
+ inactiveSecret := testSecrets.MustGetField("CODA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coda,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coda secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coda,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coda,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coda,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Coda.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Coda.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coda/coda_test.go b/pkg/detectors/coda/coda_test.go
index e260351d3..217b783ad 100644
--- a/pkg/detectors/coda/coda_test.go
+++ b/pkg/detectors/coda/coda_test.go
@@ -1,161 +1,90 @@
-//go:build detectors
-// +build detectors
-
package coda
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ coda_token: "64ukni4l-zub4-3coe-html-9byb40oi5i87"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "64ukni4l-zub4-3coe-html-9byb40oi5i87"
)
-func TestCoda_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CODA")
- inactiveSecret := testSecrets.MustGetField("CODA_INACTIVE")
+func TestCoda_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coda,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coda secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coda,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coda,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coda,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Coda.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Coda.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/codacy/codacy.go b/pkg/detectors/codacy/codacy.go
index 20b086a52..bbe419713 100644
--- a/pkg/detectors/codacy/codacy.go
+++ b/pkg/detectors/codacy/codacy.go
@@ -2,10 +2,11 @@ package codacy
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/codacy/codacy_integration_test.go b/pkg/detectors/codacy/codacy_integration_test.go
new file mode 100644
index 000000000..d086024ef
--- /dev/null
+++ b/pkg/detectors/codacy/codacy_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package codacy
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCodacy_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CODACY")
+ inactiveSecret := testSecrets.MustGetField("CODACY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codacy secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codacy,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codacy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Codacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Codacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/codacy/codacy_test.go b/pkg/detectors/codacy/codacy_test.go
index d086024ef..f416a30e3 100644
--- a/pkg/detectors/codacy/codacy_test.go
+++ b/pkg/detectors/codacy/codacy_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package codacy
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ codacy_token: "g73RSmTIzTU1wUA5BXYI"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "g73RSmTIzTU1wUA5BXYI"
)
-func TestCodacy_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CODACY")
- inactiveSecret := testSecrets.MustGetField("CODACY_INACTIVE")
+func TestCodacy_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codacy secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codacy,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codacy,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Codacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Codacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/codeclimate/codeclimate.go b/pkg/detectors/codeclimate/codeclimate.go
index 8916cc2de..9b7e4055a 100644
--- a/pkg/detectors/codeclimate/codeclimate.go
+++ b/pkg/detectors/codeclimate/codeclimate.go
@@ -4,10 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/codeclimate/codeclimate_integration_test.go b/pkg/detectors/codeclimate/codeclimate_integration_test.go
new file mode 100644
index 000000000..3a2fcf1b1
--- /dev/null
+++ b/pkg/detectors/codeclimate/codeclimate_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package codeclimate
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCodeclimate_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CODECLIMATE")
+ inactiveSecret := testSecrets.MustGetField("CODECLIMATE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codeclimate,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codeclimate,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Codeclimate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Codeclimate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/codeclimate/codeclimate_test.go b/pkg/detectors/codeclimate/codeclimate_test.go
index 2bbffcb25..2a4c6b0ac 100644
--- a/pkg/detectors/codeclimate/codeclimate_test.go
+++ b/pkg/detectors/codeclimate/codeclimate_test.go
@@ -2,116 +2,90 @@ package codeclimate
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.example.com/v1/user"
+ codeclimate_token: "efbc069555c703d31c3bcc6fbd426cec5f21eb43"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "efbc069555c703d31c3bcc6fbd426cec5f21eb43"
)
-func TestCodeclimate_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CODECLIMATE")
- inactiveSecret := testSecrets.MustGetField("CODECLIMATE_INACTIVE")
+func TestCodeClimate_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codeclimate,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codeclimate,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Codeclimate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Codeclimate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/codemagic/codemagic.go b/pkg/detectors/codemagic/codemagic.go
index 499137292..561f7ba13 100644
--- a/pkg/detectors/codemagic/codemagic.go
+++ b/pkg/detectors/codemagic/codemagic.go
@@ -2,10 +2,11 @@ package codemagic
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/codemagic/codemagic_integration_test.go b/pkg/detectors/codemagic/codemagic_integration_test.go
new file mode 100644
index 000000000..e3ab148ae
--- /dev/null
+++ b/pkg/detectors/codemagic/codemagic_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package codemagic
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCodemagic_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CODEMAGIC")
+ inactiveSecret := testSecrets.MustGetField("CODEMAGIC_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codemagic secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codemagic,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codemagic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codemagic,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Codemagic.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Codemagic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/codemagic/codemagic_test.go b/pkg/detectors/codemagic/codemagic_test.go
index e3ab148ae..e110566af 100644
--- a/pkg/detectors/codemagic/codemagic_test.go
+++ b/pkg/detectors/codemagic/codemagic_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package codemagic
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Auth-Key"
+ base_url: "https://api.example.com/v1/user"
+ codemagic_key: "PSIYbVgfkbEPQoqJfzHpACTtihONkQ_cKmOpNDPNiCU"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "PSIYbVgfkbEPQoqJfzHpACTtihONkQ_cKmOpNDPNiCU"
)
-func TestCodemagic_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CODEMAGIC")
- inactiveSecret := testSecrets.MustGetField("CODEMAGIC_INACTIVE")
+func TestCodeMagic_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codemagic secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codemagic,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codemagic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codemagic,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Codemagic.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Codemagic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/codequiry/codequiry.go b/pkg/detectors/codequiry/codequiry.go
index 0c5b64d36..94dbe6e27 100644
--- a/pkg/detectors/codequiry/codequiry.go
+++ b/pkg/detectors/codequiry/codequiry.go
@@ -2,10 +2,11 @@ package codequiry
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/codequiry/codequiry_integration_test.go b/pkg/detectors/codequiry/codequiry_integration_test.go
new file mode 100644
index 000000000..30e9e1751
--- /dev/null
+++ b/pkg/detectors/codequiry/codequiry_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package codequiry
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCodequiry_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CODEQUIRY")
+ inactiveSecret := testSecrets.MustGetField("CODEQUIRY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codequiry secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codequiry,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a codequiry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Codequiry,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Codequiry.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Codequiry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/codequiry/codequiry_test.go b/pkg/detectors/codequiry/codequiry_test.go
index 30e9e1751..268555c38 100644
--- a/pkg/detectors/codequiry/codequiry_test.go
+++ b/pkg/detectors/codequiry/codequiry_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package codequiry
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.example.com/v1/user"
+ codequiry_key: "7cA6eb3AvmlVSqLMP4iKvp7fXtEAADQud11KidjPzbSLcvntAD8CK6P7uGaGdlit"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "7cA6eb3AvmlVSqLMP4iKvp7fXtEAADQud11KidjPzbSLcvntAD8CK6P7uGaGdlit"
)
-func TestCodequiry_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CODEQUIRY")
- inactiveSecret := testSecrets.MustGetField("CODEQUIRY_INACTIVE")
+func TestCodeQuiry_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codequiry secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codequiry,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a codequiry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Codequiry,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Codequiry.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Codequiry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/coinapi/coinapi.go b/pkg/detectors/coinapi/coinapi.go
index df7886f79..ad53f8916 100644
--- a/pkg/detectors/coinapi/coinapi.go
+++ b/pkg/detectors/coinapi/coinapi.go
@@ -2,10 +2,11 @@ package coinapi
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/coinapi/coinapi_integration_test.go b/pkg/detectors/coinapi/coinapi_integration_test.go
new file mode 100644
index 000000000..9ee35f7f9
--- /dev/null
+++ b/pkg/detectors/coinapi/coinapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package coinapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoinApi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COINAPI")
+ inactiveSecret := testSecrets.MustGetField("COINAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CoinApi,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CoinApi,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CoinApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CoinApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coinapi/coinapi_test.go b/pkg/detectors/coinapi/coinapi_test.go
index 9ee35f7f9..a13dd6ce0 100644
--- a/pkg/detectors/coinapi/coinapi_test.go
+++ b/pkg/detectors/coinapi/coinapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package coinapi
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.example.com/v1/user"
+ coinapi_key: "6D8B5AUIRDQCB3NRKSMZZL9RCV9G07GTHUR3"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "6D8B5AUIRDQCB3NRKSMZZL9RCV9G07GTHUR3"
)
-func TestCoinApi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COINAPI")
- inactiveSecret := testSecrets.MustGetField("COINAPI_INACTIVE")
+func TestCoinAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coinapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CoinApi,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coinapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CoinApi,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CoinApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CoinApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/coinbase/coinbase.go b/pkg/detectors/coinbase/coinbase.go
index c1eabc672..81db4a4df 100644
--- a/pkg/detectors/coinbase/coinbase.go
+++ b/pkg/detectors/coinbase/coinbase.go
@@ -3,10 +3,11 @@ package coinbase
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/coinbase/coinbase_integration_test.go b/pkg/detectors/coinbase/coinbase_integration_test.go
new file mode 100644
index 000000000..12a2cfa39
--- /dev/null
+++ b/pkg/detectors/coinbase/coinbase_integration_test.go
@@ -0,0 +1,116 @@
+//go:build detectors
+// +build detectors
+
+package coinbase
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoinbase_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COINBASE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("COINBASE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinbase secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coinbase,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinbase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coinbase,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Coinbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Coinbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coinbase/coinbase_test.go b/pkg/detectors/coinbase/coinbase_test.go
index 5bb99e187..430c41899 100644
--- a/pkg/detectors/coinbase/coinbase_test.go
+++ b/pkg/detectors/coinbase/coinbase_test.go
@@ -1,116 +1,91 @@
-//go:build detectors
-// +build detectors
-
package coinbase
-// import (
-// "context"
-// "fmt"
-// "testing"
-// "time"
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ base_url: "https://api.example.com/v1/user"
+ coinbase_key: "IIQle6bXgoQEzxqHBt2VN0gW-Yve3t5k5VVD3nNAUUMCLVdK-3M4k6apjs43l0nM"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "IIQle6bXgoQEzxqHBt2VN0gW-Yve3t5k5VVD3nNAUUMCLVdK-3M4k6apjs43l0nM"
+)
+
+func TestCoinBase_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
+ tests := []struct {
+ name string
+ input string
+ want []string
+ }{
+ {
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
+ },
+ }
-// "github.com/kylelemons/godebug/pretty"
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
-// "github.com/trufflesecurity/trufflehog/v3/pkg/common"
-// "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-// "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-// )
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
-// func TestCoinbase_FromChunk(t *testing.T) {
-// ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-// defer cancel()
-// testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
-// if err != nil {
-// t.Fatalf("could not get test secrets from GCP: %s", err)
-// }
-// secret := testSecrets.MustGetField("COINBASE_TOKEN")
-// inactiveSecret := testSecrets.MustGetField("COINBASE_INACTIVE")
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
-// type args struct {
-// ctx context.Context
-// data []byte
-// verify bool
-// }
-// tests := []struct {
-// name string
-// s Scanner
-// args args
-// want []detectors.Result
-// wantErr bool
-// }{
-// {
-// name: "found, verified",
-// s: Scanner{},
-// args: args{
-// ctx: context.Background(),
-// data: []byte(fmt.Sprintf("You can find a coinbase secret %s within", secret)),
-// verify: true,
-// },
-// want: []detectors.Result{
-// {
-// DetectorType: detectorspb.DetectorType_Coinbase,
-// Verified: true,
-// },
-// },
-// wantErr: false,
-// },
-// {
-// name: "found, unverified",
-// s: Scanner{},
-// args: args{
-// ctx: context.Background(),
-// data: []byte(fmt.Sprintf(""You can find a coinbase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-// verify: true,
-// },
-// want: []detectors.Result{
-// {
-// DetectorType: detectorspb.DetectorType_Coinbase,
-// Verified: false,
-// },
-// },
-// wantErr: false,
-// },
-// {
-// name: "not found",
-// s: Scanner{},
-// args: args{
-// ctx: context.Background(),
-// data: []byte("You cannot find the secret within"),
-// verify: true,
-// },
-// want: nil,
-// wantErr: false,
-// },
-// }
-// for _, tt := range tests {
-// t.Run(tt.name, func(t *testing.T) {
-// s := Scanner{}
-// got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-// if (err != nil) != tt.wantErr {
-// t.Errorf("Coinbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
-// return
-// }
-// for i := range got {
-// if len(got[i].Raw) == 0 {
-// t.Fatal("no raw secret present")
-// }
-// got[i].Raw = nil
-// }
-// if diff := pretty.Compare(got, tt.want); diff != "" {
-// t.Errorf("Coinbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-// }
-// })
-// }
-// }
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
-// func BenchmarkFromData(benchmark *testing.B) {
-// ctx := context.Background()
-// s := Scanner{}
-// for name, data := range detectors.MustGetBenchmarkData() {
-// benchmark.Run(name, func(b *testing.B) {
-// for n := 0; n < b.N; n++ {
-// s.FromData(ctx, false, data)
-// }
-// })
-// }
-// }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coinlayer/coinlayer.go b/pkg/detectors/coinlayer/coinlayer.go
index ade604ad3..ba043d25c 100644
--- a/pkg/detectors/coinlayer/coinlayer.go
+++ b/pkg/detectors/coinlayer/coinlayer.go
@@ -3,11 +3,12 @@ package coinlayer
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/coinlayer/coinlayer_integration_test.go b/pkg/detectors/coinlayer/coinlayer_integration_test.go
new file mode 100644
index 000000000..095501355
--- /dev/null
+++ b/pkg/detectors/coinlayer/coinlayer_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package coinlayer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoinlayer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COINLAYER")
+ inactiveSecret := testSecrets.MustGetField("COINLAYER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coinlayer,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coinlayer,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Coinlayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Coinlayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coinlayer/coinlayer_test.go b/pkg/detectors/coinlayer/coinlayer_test.go
index 095501355..15960cce9 100644
--- a/pkg/detectors/coinlayer/coinlayer_test.go
+++ b/pkg/detectors/coinlayer/coinlayer_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package coinlayer
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.coinlayer.com/v1/user?key=gg2einqoe3zxu0ju7c3wqg9vql662vdj"
+ key: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "gg2einqoe3zxu0ju7c3wqg9vql662vdj"
)
-func TestCoinlayer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COINLAYER")
- inactiveSecret := testSecrets.MustGetField("COINLAYER_INACTIVE")
+func TestCoinLayer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coinlayer,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coinlayer,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Coinlayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Coinlayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/coinlib/coinlib.go b/pkg/detectors/coinlib/coinlib.go
index aca9a734c..92053ed40 100644
--- a/pkg/detectors/coinlib/coinlib.go
+++ b/pkg/detectors/coinlib/coinlib.go
@@ -3,10 +3,11 @@ package coinlib
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/coinlib/coinlib_integration_test.go b/pkg/detectors/coinlib/coinlib_integration_test.go
new file mode 100644
index 000000000..a49554487
--- /dev/null
+++ b/pkg/detectors/coinlib/coinlib_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package coinlib
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoinlib_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COINLIB")
+ inactiveSecret := testSecrets.MustGetField("COINLIB_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinlib secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coinlib,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coinlib secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coinlib,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Coinlib.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Coinlib.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coinlib/coinlib_test.go b/pkg/detectors/coinlib/coinlib_test.go
index bdcc00c2b..eba2c008a 100644
--- a/pkg/detectors/coinlib/coinlib_test.go
+++ b/pkg/detectors/coinlib/coinlib_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package coinlib
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.coinlib.com/v1/user?key=seugeupfknprstoe"
+ key: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "seugeupfknprstoe"
)
-func TestCoinlib_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COINLIB")
- inactiveSecret := testSecrets.MustGetField("COINLIB_INACTIVE")
+func TestCoinLib_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coinlib secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coinlib,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coinlib secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coinlib,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Coinlib.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Coinlib.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/collect2/collect2_integration_test.go b/pkg/detectors/collect2/collect2_integration_test.go
new file mode 100644
index 000000000..ddd91500e
--- /dev/null
+++ b/pkg/detectors/collect2/collect2_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package collect2
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCollect2_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COLLECT2")
+ inactiveSecret := testSecrets.MustGetField("COLLECT2_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a collect2 secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Collect2,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a collect2 secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Collect2,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Collect2.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Collect2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/collect2/collect2_test.go b/pkg/detectors/collect2/collect2_test.go
index 64f058a09..c65c7e1dc 100644
--- a/pkg/detectors/collect2/collect2_test.go
+++ b/pkg/detectors/collect2/collect2_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package collect2
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ base_url: "https://api.collect2.com/v1/user?key=22f39f53-3bd4-d84b-8e29-00402d5c316f"
+ key: ""
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "22f39f53-3bd4-d84b-8e29-00402d5c316f"
)
-func TestCollect2_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COLLECT2")
- inactiveSecret := testSecrets.MustGetField("COLLECT2_INACTIVE")
+func TestCollect2_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a collect2 secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Collect2,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a collect2 secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Collect2,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Collect2.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Collect2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/column/column_integration_test.go b/pkg/detectors/column/column_integration_test.go
new file mode 100644
index 000000000..6e9b22e4f
--- /dev/null
+++ b/pkg/detectors/column/column_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package column
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestColumn_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COLUMN_ACTIVE")
+ inactiveSecret := testSecrets.MustGetField("COLUMN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a column secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Column,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a column secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Column,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Column.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Column.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/column/column_test.go b/pkg/detectors/column/column_test.go
index 6e9b22e4f..8f19087d3 100644
--- a/pkg/detectors/column/column_test.go
+++ b/pkg/detectors/column/column_test.go
@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
package column
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestColumn_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COLUMN_ACTIVE")
- inactiveSecret := testSecrets.MustGetField("COLUMN_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Basic"
+ base_url: "https://api.example.com/v1/user"
+ column_key: "live_ID8Jxlu0QRsV7rKkWI9CUDpkrUv"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "live_ID8Jxlu0QRsV7rKkWI9CUDpkrUv"
+)
+
+func TestColumn_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a column secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Column,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a column secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Column,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Column.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Column.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/commercejs/commercejs.go b/pkg/detectors/commercejs/commercejs.go
index d7af6292a..781fd4591 100644
--- a/pkg/detectors/commercejs/commercejs.go
+++ b/pkg/detectors/commercejs/commercejs.go
@@ -2,10 +2,11 @@ package commercejs
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/commercejs/commercejs_integration_test.go b/pkg/detectors/commercejs/commercejs_integration_test.go
new file mode 100644
index 000000000..0c39e067e
--- /dev/null
+++ b/pkg/detectors/commercejs/commercejs_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package commercejs
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCommerceJS_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COMMERCEJS")
+ inactiveSecret := testSecrets.MustGetField("COMMERCEJS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a commercejs secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CommerceJS,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a commercejs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CommerceJS,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CommerceJS.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CommerceJS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/commercejs/commercejs_test.go b/pkg/detectors/commercejs/commercejs_test.go
index 0c39e067e..57363764a 100644
--- a/pkg/detectors/commercejs/commercejs_test.go
+++ b/pkg/detectors/commercejs/commercejs_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package commercejs
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ in: "Header"
+ base_url: "https://api.example.com/v1/user"
+ commercejs_key: "g6cl4jt_2noyibalgbqid4h58jxivqdgyyxovepbvqmbl7wq"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "g6cl4jt_2noyibalgbqid4h58jxivqdgyyxovepbvqmbl7wq"
)
-func TestCommerceJS_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COMMERCEJS")
- inactiveSecret := testSecrets.MustGetField("COMMERCEJS_INACTIVE")
+func TestCommerceJS_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a commercejs secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CommerceJS,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a commercejs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CommerceJS,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CommerceJS.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CommerceJS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/commodities/commodities_integration_test.go b/pkg/detectors/commodities/commodities_integration_test.go
new file mode 100644
index 000000000..0c0ef7c2d
--- /dev/null
+++ b/pkg/detectors/commodities/commodities_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package commodities
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCommodities_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COMMODITIES")
+ inactiveSecret := testSecrets.MustGetField("COMMODITIES_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a commodities secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Commodities,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a commodities secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Commodities,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Commodities.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Commodities.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/commodities/commodities_test.go b/pkg/detectors/commodities/commodities_test.go
index 0c0ef7c2d..cb857f603 100644
--- a/pkg/detectors/commodities/commodities_test.go
+++ b/pkg/detectors/commodities/commodities_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package commodities
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ commodities_key: "5aJKBqkCyGCT9FIWNUTmowbzqgcm9DUCi60mHwgPQRBSt7dFahv9eY329Dn9"
+ base_url: "https://api.example.com/v1/user?access_key=$commodities_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "5aJKBqkCyGCT9FIWNUTmowbzqgcm9DUCi60mHwgPQRBSt7dFahv9eY329Dn9"
)
-func TestCommodities_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COMMODITIES")
- inactiveSecret := testSecrets.MustGetField("COMMODITIES_INACTIVE")
+func TestCommodities_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a commodities secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Commodities,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a commodities secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Commodities,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Commodities.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Commodities.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/companyhub/companyhub.go b/pkg/detectors/companyhub/companyhub.go
index ead1f778e..6350835b3 100644
--- a/pkg/detectors/companyhub/companyhub.go
+++ b/pkg/detectors/companyhub/companyhub.go
@@ -3,10 +3,11 @@ package companyhub
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/companyhub/companyhub_integration_test.go b/pkg/detectors/companyhub/companyhub_integration_test.go
new file mode 100644
index 000000000..877eef42e
--- /dev/null
+++ b/pkg/detectors/companyhub/companyhub_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package companyhub
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCompanyHub_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COMPANYHUB_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("COMPANYHUB_INACTIVE")
+ user := testSecrets.MustGetField("COMPANYHUB_USER")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s", secret, user)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CompanyHub,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CompanyHub,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CompanyHub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CompanyHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/companyhub/companyhub_test.go b/pkg/detectors/companyhub/companyhub_test.go
index 877eef42e..eb2c7faf8 100644
--- a/pkg/detectors/companyhub/companyhub_test.go
+++ b/pkg/detectors/companyhub/companyhub_test.go
@@ -1,121 +1,96 @@
-//go:build detectors
-// +build detectors
-
package companyhub
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCompanyHub_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COMPANYHUB_TOKEN")
- inactiveSecret := testSecrets.MustGetField("COMPANYHUB_INACTIVE")
- user := testSecrets.MustGetField("COMPANYHUB_USER")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ api:
+ auth_type: ""
+ in: "Header"
+ companyhub_key: "A5zrYt9xY4X1Q9mG6IX6"
+ companyhub_id: "xzAMMncOTR7^5d5NS6"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secrets = []string{
+ "A5zrYt9xY4X1Q9mG6IX6xzAMMncOTR7^5d5NS6",
+ "A5zrYt9xY4X1Q9mG6IX6A5zrYt9xY4X1Q9mG6IX6",
}
+)
+
+func TestCompanyHub_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s", secret, user)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CompanyHub,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CompanyHub,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: secrets,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CompanyHub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CompanyHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/confluent/confluent.go b/pkg/detectors/confluent/confluent.go
index 82d34e52f..ea73d2bd8 100644
--- a/pkg/detectors/confluent/confluent.go
+++ b/pkg/detectors/confluent/confluent.go
@@ -4,16 +4,17 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/confluent/confluent_integration_test.go b/pkg/detectors/confluent/confluent_integration_test.go
new file mode 100644
index 000000000..91b535dec
--- /dev/null
+++ b/pkg/detectors/confluent/confluent_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package confluent
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestConfluent_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CONFLUENT_TOKEN")
+ key := testSecrets.MustGetField("CONFLUENT_KEY")
+ inactiveSecret := testSecrets.MustGetField("CONFLUENT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s", secret, key)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Confluent,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Confluent,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Confluent.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Confluent.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/confluent/confluent_test.go b/pkg/detectors/confluent/confluent_test.go
index 91b535dec..2d541a112 100644
--- a/pkg/detectors/confluent/confluent_test.go
+++ b/pkg/detectors/confluent/confluent_test.go
@@ -1,121 +1,93 @@
-//go:build detectors
-// +build detectors
-
package confluent
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Basic"
+ in: "Header"
+ confluent_key: "CVAJHB4RAZboV3Od"
+ confluent_secret: "pIsdFuG0oJuyiir3GWqpC4pv7xpKFodCNh6PYN4XdE8EtyIwYtzEer0KtHQ8kofs"
+ base_url: "https://api.example.com/v1/user?api-key=$confluent_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "CVAJHB4RAZboV3OdpIsdFuG0oJuyiir3GWqpC4pv7xpKFodCNh6PYN4XdE8EtyIwYtzEer0KtHQ8kofs"
)
-func TestConfluent_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CONFLUENT_TOKEN")
- key := testSecrets.MustGetField("CONFLUENT_KEY")
- inactiveSecret := testSecrets.MustGetField("CONFLUENT_INACTIVE")
+func TestConfluent_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s", secret, key)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Confluent,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Confluent,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Confluent.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Confluent.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go
index ecd70e57a..e8004a608 100644
--- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go
+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go
@@ -3,10 +3,11 @@ package contentfulpersonalaccesstoken
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go
index f69286595..52c2b55b5 100644
--- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go
+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go
@@ -1,121 +1,91 @@
-//go:build detectors
-// +build detectors
-
package contentfulpersonalaccesstoken
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ in: "Header"
+ contentful_access_token: "CFPAT-"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
)
-func TestContentfulPersonalAccessToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN")
- inactiveSecret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN_INACTIVE")
+func TestContentfulPersonalAccessToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a contentful secret %s within contentful https://api.contentful.com/organizations", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a contentful secret %s within but unverified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - not found",
+ input: validPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ContentfulPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ContentfulPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalacesstoken_integration_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalacesstoken_integration_test.go
new file mode 100644
index 000000000..f69286595
--- /dev/null
+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalacesstoken_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package contentfulpersonalaccesstoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestContentfulPersonalAccessToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN")
+ inactiveSecret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a contentful secret %s within contentful https://api.contentful.com/organizations", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a contentful secret %s within but unverified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ContentfulPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+
+ t.Errorf("ContentfulPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/conversiontools/conversiontools.go b/pkg/detectors/conversiontools/conversiontools.go
index f1b012007..f4732bf38 100644
--- a/pkg/detectors/conversiontools/conversiontools.go
+++ b/pkg/detectors/conversiontools/conversiontools.go
@@ -3,10 +3,11 @@ package conversiontools
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/conversiontools/conversiontools_integration_test.go b/pkg/detectors/conversiontools/conversiontools_integration_test.go
new file mode 100644
index 000000000..ab698d0a2
--- /dev/null
+++ b/pkg/detectors/conversiontools/conversiontools_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package conversiontools
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestConversionTools_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CONVERSIONTOOLS")
+ inactiveSecret := testSecrets.MustGetField("CONVERSIONTOOLS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ConversionTools,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ConversionTools,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ConversionTools.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ConversionTools.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/conversiontools/conversiontools_test.go b/pkg/detectors/conversiontools/conversiontools_test.go
index 748c3567c..ff0893a10 100644
--- a/pkg/detectors/conversiontools/conversiontools_test.go
+++ b/pkg/detectors/conversiontools/conversiontools_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package conversiontools
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ in: "Header"
+ conversiontools_key: "ey5g5C73ichf2TWwQQfuPNG2SW1xdTmCHFgS6zsUjRz3kkiEofoa8X7SVGjwAMkhrv5KyOFqunP29gQpKq9A4sPF_Ps4B4IkTtgUG9cgP5A5ygAkuSR2rsOC.SIDSLIy4jZiL7L8ZHyAhyR8msV7JzxlI6YsNqmmj"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "ey5g5C73ichf2TWwQQfuPNG2SW1xdTmCHFgS6zsUjRz3kkiEofoa8X7SVGjwAMkhrv5KyOFqunP29gQpKq9A4sPF_Ps4B4IkTtgUG9cgP5A5ygAkuSR2rsOC.SIDSLIy4jZiL7L8ZHyAhyR8msV7JzxlI6YsNqmmj"
)
-func TestConversionTools_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CONVERSIONTOOLS")
- inactiveSecret := testSecrets.MustGetField("CONVERSIONTOOLS_INACTIVE")
+func TestConversionTools_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ConversionTools,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ConversionTools,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ConversionTools.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ConversionTools.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/convertapi/convertapi.go b/pkg/detectors/convertapi/convertapi.go
index 600ea2dbe..06dbe7d1b 100644
--- a/pkg/detectors/convertapi/convertapi.go
+++ b/pkg/detectors/convertapi/convertapi.go
@@ -3,10 +3,11 @@ package convertapi
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/convertapi/convertapi_integration_test.go b/pkg/detectors/convertapi/convertapi_integration_test.go
new file mode 100644
index 000000000..89acb1db5
--- /dev/null
+++ b/pkg/detectors/convertapi/convertapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package convertapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestConvertApi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CONVERTAPI")
+ inactiveSecret := testSecrets.MustGetField("CONVERTAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a convertapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ConvertApi,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a convertapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ConvertApi,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ConvertApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ConvertApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/convertapi/convertapi_test.go b/pkg/detectors/convertapi/convertapi_test.go
index 89acb1db5..869a552d6 100644
--- a/pkg/detectors/convertapi/convertapi_test.go
+++ b/pkg/detectors/convertapi/convertapi_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package convertapi
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ convertapi_key: "H9ZGTfAERfN5W0AX"
+ base_url: "https://api.example.com/v1/user?Secret=$convertapi_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "H9ZGTfAERfN5W0AX"
)
-func TestConvertApi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CONVERTAPI")
- inactiveSecret := testSecrets.MustGetField("CONVERTAPI_INACTIVE")
+func TestConvertAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a convertapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ConvertApi,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a convertapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ConvertApi,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ConvertApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ConvertApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/convertkit/convertkit.go b/pkg/detectors/convertkit/convertkit.go
index 1f915d8a2..35cb1f139 100644
--- a/pkg/detectors/convertkit/convertkit.go
+++ b/pkg/detectors/convertkit/convertkit.go
@@ -2,10 +2,11 @@ package convertkit
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/convertkit/convertkit_integration_test.go b/pkg/detectors/convertkit/convertkit_integration_test.go
new file mode 100644
index 000000000..2b773d28c
--- /dev/null
+++ b/pkg/detectors/convertkit/convertkit_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package convertkit
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestConvertkit_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CONVERTKIT_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CONVERTKIT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a convertkit secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Convertkit,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a convertkit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Convertkit,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Convertkit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Convertkit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/convertkit/convertkit_test.go b/pkg/detectors/convertkit/convertkit_test.go
index 2b773d28c..5909f9f5b 100644
--- a/pkg/detectors/convertkit/convertkit_test.go
+++ b/pkg/detectors/convertkit/convertkit_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package convertkit
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ convertkit_key: "hfCnuVcYOgiRjlDEmAoRbN"
+ base_url: "https://api.example.com/v1/forms?api-key=$convertapi_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "hfCnuVcYOgiRjlDEmAoRbN"
)
-func TestConvertkit_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CONVERTKIT_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CONVERTKIT_INACTIVE")
+func TestConvertKit_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a convertkit secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Convertkit,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a convertkit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Convertkit,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Convertkit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Convertkit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/convier/convier_integration_test.go b/pkg/detectors/convier/convier_integration_test.go
new file mode 100644
index 000000000..190274a00
--- /dev/null
+++ b/pkg/detectors/convier/convier_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package convier
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestConvier_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CONVIER")
+ inactiveSecret := testSecrets.MustGetField("CONVIER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a convier secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Convier,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a convier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Convier,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Convier.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Convier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/convier/convier_test.go b/pkg/detectors/convier/convier_test.go
index 190274a00..86d1d76b3 100644
--- a/pkg/detectors/convier/convier_test.go
+++ b/pkg/detectors/convier/convier_test.go
@@ -1,116 +1,91 @@
-//go:build detectors
-// +build detectors
-
package convier
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestConvier_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CONVIER")
- inactiveSecret := testSecrets.MustGetField("CONVIER_INACTIVE")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ in: "Header"
+ convier_key: "49|07KJBwfPzF2ESyNui5yBw9OVB6eWj0iXkssEKC7b"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "49|07KJBwfPzF2ESyNui5yBw9OVB6eWj0iXkssEKC7b"
+)
+
+func TestConvier_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a convier secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Convier,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a convier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Convier,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Convier.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Convier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/copper/copper.go b/pkg/detectors/copper/copper.go
index 60ca33216..ff1ec3c95 100644
--- a/pkg/detectors/copper/copper.go
+++ b/pkg/detectors/copper/copper.go
@@ -2,10 +2,11 @@ package copper
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/copper/copper_integration_test.go b/pkg/detectors/copper/copper_integration_test.go
new file mode 100644
index 000000000..9849e144a
--- /dev/null
+++ b/pkg/detectors/copper/copper_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package copper
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCopper_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COOPER_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("COOPER_INACTIVE")
+ id := testSecrets.MustGetField("COOPER_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Copper,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Copper,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Copper.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Copper.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/copper/copper_test.go b/pkg/detectors/copper/copper_test.go
index cf7d9f021..e6f9944ef 100644
--- a/pkg/detectors/copper/copper_test.go
+++ b/pkg/detectors/copper/copper_test.go
@@ -1,121 +1,93 @@
-//go:build detectors
-// +build detectors
-
package copper
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ in: "Header"
+ copper_email: "s0ovh@P8I~p3"
+ copper_token: "noqs39jzqaegbam2k6mai9ov1uwsl21y"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "noqs39jzqaegbam2k6mai9ov1uwsl21ys0ovh@P8I~p3"
)
-func TestCopper_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COOPER_TOKEN")
- inactiveSecret := testSecrets.MustGetField("COOPER_INACTIVE")
- id := testSecrets.MustGetField("COOPER_ID")
+func TestCopper_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Copper,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Copper,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Copper.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Copper.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/couchbase/couchbase.go b/pkg/detectors/couchbase/couchbase.go
index 43e60e3de..2d47de59c 100644
--- a/pkg/detectors/couchbase/couchbase.go
+++ b/pkg/detectors/couchbase/couchbase.go
@@ -15,7 +15,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/couchbase/couchbase_integration_test.go b/pkg/detectors/couchbase/couchbase_integration_test.go
new file mode 100644
index 000000000..03b340879
--- /dev/null
+++ b/pkg/detectors/couchbase/couchbase_integration_test.go
@@ -0,0 +1,123 @@
+//go:build detectors
+// +build detectors
+
+package couchbase
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCouchbase_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")
+ username := testSecrets.MustGetField("COUCHBASE_USERNAME")
+ password := testSecrets.MustGetField("COUCHBASE_PASSWORD")
+ inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Couchbase,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Couchbase,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/couchbase/couchbase_test.go b/pkg/detectors/couchbase/couchbase_test.go
index 03b340879..9412bdbad 100644
--- a/pkg/detectors/couchbase/couchbase_test.go
+++ b/pkg/detectors/couchbase/couchbase_test.go
@@ -1,123 +1,97 @@
-//go:build detectors
-// +build detectors
-
package couchbase
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCouchbase_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")
- username := testSecrets.MustGetField("COUCHBASE_USERNAME")
- password := testSecrets.MustGetField("COUCHBASE_PASSWORD")
- inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Password"
+ in: "Configuration"
+ couchbase_domain: "cb.testing.cloud.couchbase.com" // couchbase://
+ couchbase_username: "usrpS@d>p"
+ couchbase_password: "passwordU+2028 rf\@V[4,L/?2}"
+ base_url: "https://$couchbase_domain/v1/user"
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secrets = []string{
+ `usrpS@d>p:passwordU+2028@couchbases://cb.testing.cloud.couchbase.com`,
+ `$DB_USERNAME:passwordU+2028@couchbases://cb.testing.cloud.couchbase.com`,
}
+)
+
+func TestCouchBase_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Couchbase,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Couchbase,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: secrets,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/countrylayer/countrylayer.go b/pkg/detectors/countrylayer/countrylayer.go
index cee4ca102..57a402597 100644
--- a/pkg/detectors/countrylayer/countrylayer.go
+++ b/pkg/detectors/countrylayer/countrylayer.go
@@ -3,10 +3,11 @@ package countrylayer
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/countrylayer/countrylayer_integration_test.go b/pkg/detectors/countrylayer/countrylayer_integration_test.go
new file mode 100644
index 000000000..c70dcc65a
--- /dev/null
+++ b/pkg/detectors/countrylayer/countrylayer_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package countrylayer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCountryLayer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COUNTRYLAYER")
+ inactiveSecret := testSecrets.MustGetField("COUNTRYLAYER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CountryLayer,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CountryLayer,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CountryLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CountryLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/countrylayer/countrylayer_test.go b/pkg/detectors/countrylayer/countrylayer_test.go
index c70dcc65a..f97fd88cd 100644
--- a/pkg/detectors/countrylayer/countrylayer_test.go
+++ b/pkg/detectors/countrylayer/countrylayer_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package countrylayer
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ countrylayer_key: "031eiaqplnq39py5ppsctxo6n2xj5t10"
+ base_url: "https://api.example.com/v1/user?access_key=$countrylayer_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "031eiaqplnq39py5ppsctxo6n2xj5t10"
)
-func TestCountryLayer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COUNTRYLAYER")
- inactiveSecret := testSecrets.MustGetField("COUNTRYLAYER_INACTIVE")
+func TestCountryLayer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CountryLayer,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CountryLayer,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CountryLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CountryLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/courier/courier.go b/pkg/detectors/courier/courier.go
index 822fe2e2c..96db0c1c1 100644
--- a/pkg/detectors/courier/courier.go
+++ b/pkg/detectors/courier/courier.go
@@ -3,10 +3,11 @@ package courier
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/courier/courier_integration_test.go b/pkg/detectors/courier/courier_integration_test.go
new file mode 100644
index 000000000..2b6f81803
--- /dev/null
+++ b/pkg/detectors/courier/courier_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package courier
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCourier_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COURIER")
+ inactiveSecret := testSecrets.MustGetField("COURIER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a courier secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Courier,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a courier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Courier,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Courier.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Courier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/courier/courier_test.go b/pkg/detectors/courier/courier_test.go
index 2b6f81803..b7c90a976 100644
--- a/pkg/detectors/courier/courier_test.go
+++ b/pkg/detectors/courier/courier_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package courier
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ in: "Header"
+ courier_key: "pk_iHWk6NqTne0QthfSVF7uixZpa3OTYpA8hC6bIIavhluXfxz37FB_rHPqJNWh06HpNIOuokET4dfFzXS1"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "pk_iHWk6NqTne0QthfSVF7uixZpa3OTYpA8hC6bIIavhluXfxz37FB_rHPqJNWh06HpNIOuokET4dfFzXS1"
)
-func TestCourier_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COURIER")
- inactiveSecret := testSecrets.MustGetField("COURIER_INACTIVE")
+func TestCourier_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a courier secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Courier,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a courier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Courier,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Courier.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Courier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/coveralls/coveralls.go b/pkg/detectors/coveralls/coveralls.go
index 17e412fb0..f22fb29e3 100644
--- a/pkg/detectors/coveralls/coveralls.go
+++ b/pkg/detectors/coveralls/coveralls.go
@@ -3,10 +3,11 @@ package coveralls
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/coveralls/coveralls_integration_test.go b/pkg/detectors/coveralls/coveralls_integration_test.go
new file mode 100644
index 000000000..17b7299de
--- /dev/null
+++ b/pkg/detectors/coveralls/coveralls_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package coveralls
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCoveralls_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("COVERALLS_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("COVERALLS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coveralls secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coveralls,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a coveralls secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Coveralls,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Coveralls.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Coveralls.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/coveralls/coveralls_test.go b/pkg/detectors/coveralls/coveralls_test.go
index 2b20f8ece..32e36173e 100644
--- a/pkg/detectors/coveralls/coveralls_test.go
+++ b/pkg/detectors/coveralls/coveralls_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package coveralls
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ in: "Header"
+ coveralls_token: "tPfhjkzKJyWtUdxDLYMjNDEfP7Yn9WvWb2-K3"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "tPfhjkzKJyWtUdxDLYMjNDEfP7Yn9WvWb2-K3"
)
-func TestCoveralls_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("COVERALLS_TOKEN")
- inactiveSecret := testSecrets.MustGetField("COVERALLS_INACTIVE")
+func TestCoveralls_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coveralls secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coveralls,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a coveralls secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Coveralls,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Coveralls.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Coveralls.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/craftmypdf/craftmypdf_integration_test.go b/pkg/detectors/craftmypdf/craftmypdf_integration_test.go
new file mode 100644
index 000000000..b2952d214
--- /dev/null
+++ b/pkg/detectors/craftmypdf/craftmypdf_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package craftmypdf
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCraftMyPDF_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CRAFTMYPDF")
+ inactiveSecret := testSecrets.MustGetField("CRAFTMYPDF_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CraftMyPDF,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CraftMyPDF,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CraftMyPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CraftMyPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/craftmypdf/craftmypdf_test.go b/pkg/detectors/craftmypdf/craftmypdf_test.go
index b2952d214..2df3db9f5 100644
--- a/pkg/detectors/craftmypdf/craftmypdf_test.go
+++ b/pkg/detectors/craftmypdf/craftmypdf_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package craftmypdf
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Header"
+ craftmypdf_key: "GuTSS3XQdT6fx00mxudKq7oj2CsieZCGmEc"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "GuTSS3XQdT6fx00mxudKq7oj2CsieZCGmEc"
)
-func TestCraftMyPDF_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CRAFTMYPDF")
- inactiveSecret := testSecrets.MustGetField("CRAFTMYPDF_INACTIVE")
+func TestCraftMyPDF_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CraftMyPDF,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CraftMyPDF,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CraftMyPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CraftMyPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/crowdin/crowdin.go b/pkg/detectors/crowdin/crowdin.go
index 57875afc1..ff3073cf6 100644
--- a/pkg/detectors/crowdin/crowdin.go
+++ b/pkg/detectors/crowdin/crowdin.go
@@ -3,10 +3,11 @@ package crowdin
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/crowdin/crowdin_integration_test.go b/pkg/detectors/crowdin/crowdin_integration_test.go
new file mode 100644
index 000000000..10a050d8c
--- /dev/null
+++ b/pkg/detectors/crowdin/crowdin_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package crowdin
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCrowdin_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CROWDIN")
+ inactiveSecret := testSecrets.MustGetField("CROWDIN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a crowdin secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Crowdin,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a crowdin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Crowdin,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Crowdin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Crowdin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/crowdin/crowdin_test.go b/pkg/detectors/crowdin/crowdin_test.go
index 10a050d8c..fab9962c4 100644
--- a/pkg/detectors/crowdin/crowdin_test.go
+++ b/pkg/detectors/crowdin/crowdin_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package crowdin
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "Bearer"
+ in: "Header"
+ crowdin_token: "BiIRgdPvboWwqlhQtlnCsM041zVYCJ5yMfgltWesDiu9bv1nuRtCEPewsDL3vgRFcp2qLemaPMa8L9g7"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "BiIRgdPvboWwqlhQtlnCsM041zVYCJ5yMfgltWesDiu9bv1nuRtCEPewsDL3vgRFcp2qLemaPMa8L9g7"
)
-func TestCrowdin_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CROWDIN")
- inactiveSecret := testSecrets.MustGetField("CROWDIN_INACTIVE")
+func TestCrowDin_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a crowdin secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Crowdin,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a crowdin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Crowdin,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Crowdin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Crowdin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/cryptocompare/cryptocompare.go b/pkg/detectors/cryptocompare/cryptocompare.go
index a1cfd7f6c..02974f301 100644
--- a/pkg/detectors/cryptocompare/cryptocompare.go
+++ b/pkg/detectors/cryptocompare/cryptocompare.go
@@ -2,11 +2,12 @@ package cryptocompare
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/cryptocompare/cryptocompare_integration_test.go b/pkg/detectors/cryptocompare/cryptocompare_integration_test.go
new file mode 100644
index 000000000..49f069a3c
--- /dev/null
+++ b/pkg/detectors/cryptocompare/cryptocompare_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package cryptocompare
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCryptoCompare_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CRYPTOCOMPARE")
+ inactiveSecret := testSecrets.MustGetField("CRYPTOCOMPARE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CryptoCompare,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CryptoCompare,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CryptoCompare.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CryptoCompare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/cryptocompare/cryptocompare_test.go b/pkg/detectors/cryptocompare/cryptocompare_test.go
index 49f069a3c..9f0a319d0 100644
--- a/pkg/detectors/cryptocompare/cryptocompare_test.go
+++ b/pkg/detectors/cryptocompare/cryptocompare_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package cryptocompare
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Header"
+ cryptocompare_key: "lx8zzovs5h15zl15mj224zks2v25re59965gz0l1z4jsc0bng33a75m5pf52-bvd"
+ base_url: "https://api.example.com/v1/user?api_key=$cryptocompare_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "lx8zzovs5h15zl15mj224zks2v25re59965gz0l1z4jsc0bng33a75m5pf52-bvd"
)
-func TestCryptoCompare_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CRYPTOCOMPARE")
- inactiveSecret := testSecrets.MustGetField("CRYPTOCOMPARE_INACTIVE")
+func TestCryptoCompare_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CryptoCompare,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CryptoCompare,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CryptoCompare.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CryptoCompare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/currencyfreaks/currencyfreaks.go b/pkg/detectors/currencyfreaks/currencyfreaks.go
index b4306d84b..5efc9c7c1 100644
--- a/pkg/detectors/currencyfreaks/currencyfreaks.go
+++ b/pkg/detectors/currencyfreaks/currencyfreaks.go
@@ -2,10 +2,11 @@ package currencyfreaks
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/currencyfreaks/currencyfreaks_integration_test.go b/pkg/detectors/currencyfreaks/currencyfreaks_integration_test.go
new file mode 100644
index 000000000..e3daa56b1
--- /dev/null
+++ b/pkg/detectors/currencyfreaks/currencyfreaks_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package currencyfreaks
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCurrencyfreaks_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CURRENCYFREAKS")
+ inactiveSecret := testSecrets.MustGetField("CURRENCYFREAKS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Currencyfreaks,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Currencyfreaks,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Currencyfreaks.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Currencyfreaks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/currencyfreaks/currencyfreaks_test.go b/pkg/detectors/currencyfreaks/currencyfreaks_test.go
index e3daa56b1..8349dd400 100644
--- a/pkg/detectors/currencyfreaks/currencyfreaks_test.go
+++ b/pkg/detectors/currencyfreaks/currencyfreaks_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package currencyfreaks
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Header"
+ currencyfreaks_key: "6zlrpo4u8z4s72b2nqr54m9ehmqvwe8p"
+ base_url: "https://api.example.com/v1/user?apiKey=$currencyfreaks_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "6zlrpo4u8z4s72b2nqr54m9ehmqvwe8p"
)
-func TestCurrencyfreaks_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CURRENCYFREAKS")
- inactiveSecret := testSecrets.MustGetField("CURRENCYFREAKS_INACTIVE")
+func TestCurrencyFreaks_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Currencyfreaks,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Currencyfreaks,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Currencyfreaks.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Currencyfreaks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/currencylayer/currencylayer.go b/pkg/detectors/currencylayer/currencylayer.go
index fdede2bfe..3bb7966ce 100644
--- a/pkg/detectors/currencylayer/currencylayer.go
+++ b/pkg/detectors/currencylayer/currencylayer.go
@@ -3,11 +3,12 @@ package currencylayer
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/currencylayer/currencylayer_integration_test.go b/pkg/detectors/currencylayer/currencylayer_integration_test.go
new file mode 100644
index 000000000..5cc7958d8
--- /dev/null
+++ b/pkg/detectors/currencylayer/currencylayer_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package currencylayer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCurrencylayer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CURRENCYLAYER")
+ inactiveSecret := testSecrets.MustGetField("CURRENCYLAYER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Currencylayer,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Currencylayer,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Currencylayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Currencylayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/currencylayer/currencylayer_test.go b/pkg/detectors/currencylayer/currencylayer_test.go
index 5cc7958d8..bc8d98ea0 100644
--- a/pkg/detectors/currencylayer/currencylayer_test.go
+++ b/pkg/detectors/currencylayer/currencylayer_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package currencylayer
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ currencylayer_key: "sxthwp257vpusfe4gr4d4awc794lkxvh"
+ base_url: "https://api.example.com/v1/user?access_key=$currencylayer_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "sxthwp257vpusfe4gr4d4awc794lkxvh"
)
-func TestCurrencylayer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CURRENCYLAYER")
- inactiveSecret := testSecrets.MustGetField("CURRENCYLAYER_INACTIVE")
+func TestCurrencyLayer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Currencylayer,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Currencylayer,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Currencylayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Currencylayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/currencyscoop/currencyscoop.go b/pkg/detectors/currencyscoop/currencyscoop.go
index 539371b4a..452d29900 100644
--- a/pkg/detectors/currencyscoop/currencyscoop.go
+++ b/pkg/detectors/currencyscoop/currencyscoop.go
@@ -3,10 +3,11 @@ package currencyscoop
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/currencyscoop/currencyscoop_integration_test.go b/pkg/detectors/currencyscoop/currencyscoop_integration_test.go
new file mode 100644
index 000000000..378a11cff
--- /dev/null
+++ b/pkg/detectors/currencyscoop/currencyscoop_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package currencyscoop
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCurrencyscoop_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CURRENCYSCOOP")
+ inactiveSecret := testSecrets.MustGetField("CURRENCYSCOOP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CurrencyScoop,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CurrencyScoop,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Currencyscoop.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Currencyscoop.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/currencyscoop/currencyscoop_test.go b/pkg/detectors/currencyscoop/currencyscoop_test.go
index 378a11cff..08eebc55b 100644
--- a/pkg/detectors/currencyscoop/currencyscoop_test.go
+++ b/pkg/detectors/currencyscoop/currencyscoop_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package currencyscoop
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ currencyscoop_key: "70x6tezndca5dqlm5tnn7s03bm6c27jt"
+ base_url: "https://api.example.com/v1/user?api_key=$currencylayer_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "70x6tezndca5dqlm5tnn7s03bm6c27jt"
)
-func TestCurrencyscoop_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CURRENCYSCOOP")
- inactiveSecret := testSecrets.MustGetField("CURRENCYSCOOP_INACTIVE")
+func TestCurrencyScoop_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CurrencyScoop,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CurrencyScoop,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Currencyscoop.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Currencyscoop.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/currentsapi/currentsapi.go b/pkg/detectors/currentsapi/currentsapi.go
index 782eb7c22..ae9d3a7fe 100644
--- a/pkg/detectors/currentsapi/currentsapi.go
+++ b/pkg/detectors/currentsapi/currentsapi.go
@@ -2,10 +2,11 @@ package currentsapi
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/currentsapi/currentsapi_integration_test.go b/pkg/detectors/currentsapi/currentsapi_integration_test.go
new file mode 100644
index 000000000..63a8e8ed5
--- /dev/null
+++ b/pkg/detectors/currentsapi/currentsapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package currentsapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCurrentsAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CURRENTSAPI")
+ inactiveSecret := testSecrets.MustGetField("CURRENTSAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CurrentsAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CurrentsAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CurrentsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CurrentsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/currentsapi/currentsapi_test.go b/pkg/detectors/currentsapi/currentsapi_test.go
index 63a8e8ed5..63351932b 100644
--- a/pkg/detectors/currentsapi/currentsapi_test.go
+++ b/pkg/detectors/currentsapi/currentsapi_test.go
@@ -1,120 +1,92 @@
-//go:build detectors
-// +build detectors
-
package currentsapi
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: ""
+ in: "Header"
+ currentsapi_key: "thisistheapikey(asaspijf89aa$)lkunjo0#aenoiecnjh"
+ base_url: "https://api.example.com/v1/user"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "thisistheapikey(asaspijf89aa$)lkunjo0#aenoiecnjh"
)
-func TestCurrentsAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CURRENTSAPI")
- inactiveSecret := testSecrets.MustGetField("CURRENTSAPI_INACTIVE")
+func TestCurrentsAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CurrentsAPI,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CurrentsAPI,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CurrentsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CurrentsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/customerguru/customerguru.go b/pkg/detectors/customerguru/customerguru.go
index 1a6d87ad3..a7fe1690c 100644
--- a/pkg/detectors/customerguru/customerguru.go
+++ b/pkg/detectors/customerguru/customerguru.go
@@ -2,16 +2,17 @@ package customerguru
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
diff --git a/pkg/detectors/customerguru/customerguru_integration_test.go b/pkg/detectors/customerguru/customerguru_integration_test.go
new file mode 100644
index 000000000..3c16c76aa
--- /dev/null
+++ b/pkg/detectors/customerguru/customerguru_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package customerguru
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCustomerGuru_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CUSTOMERGURU_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CUSTOMERGURU_INACTIVE")
+ key := testSecrets.MustGetField("CUSTOMERGURU_KEY")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s", secret, key)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CustomerGuru,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CustomerGuru,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CustomerGuru.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CustomerGuru.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/customerguru/customerguru_test.go b/pkg/detectors/customerguru/customerguru_test.go
index 3c16c76aa..960a8557f 100644
--- a/pkg/detectors/customerguru/customerguru_test.go
+++ b/pkg/detectors/customerguru/customerguru_test.go
@@ -1,121 +1,93 @@
-//go:build detectors
-// +build detectors
-
package customerguru
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
+
+ api:
+ auth_type: "API-Key"
+ in: "Path"
+ guru_key: "WWj2zAK0tMkVJqc28Itfu6THQycyfT"
+ guru_id: "Ic53IHpPK71wIacbCgEkIlFbw0VIMcsz6ir2i2DJ0XDRdirf2K"
+ base_url: "https://api.customerguru.com/v1/user?api_secret=$guru_id&api_token=guru_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secret = "WWj2zAK0tMkVJqc28Itfu6THQycyfTIc53IHpPK71wIacbCgEkIlFbw0VIMcsz6ir2i2DJ0XDRdirf2K"
)
-func TestCustomerGuru_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CUSTOMERGURU_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CUSTOMERGURU_INACTIVE")
- key := testSecrets.MustGetField("CUSTOMERGURU_KEY")
+func TestCustomerGuru_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s", secret, key)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CustomerGuru,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CustomerGuru,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{secret},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CustomerGuru.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CustomerGuru.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/customerio/customerio.go b/pkg/detectors/customerio/customerio.go
index 349d4c01c..770228a23 100644
--- a/pkg/detectors/customerio/customerio.go
+++ b/pkg/detectors/customerio/customerio.go
@@ -4,10 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/customerio/customerio_integration_test.go b/pkg/detectors/customerio/customerio_integration_test.go
new file mode 100644
index 000000000..a28613e04
--- /dev/null
+++ b/pkg/detectors/customerio/customerio_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package customerio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestCustomerIO_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("CUSTOMERIO_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("CUSTOMERIO_INACTIVE")
+ id := testSecrets.MustGetField("CUSTOMERIO_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s ", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CustomerIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_CustomerIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("CustomerIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("CustomerIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/customerio/customerio_test.go b/pkg/detectors/customerio/customerio_test.go
index a28613e04..e8cbf150b 100644
--- a/pkg/detectors/customerio/customerio_test.go
+++ b/pkg/detectors/customerio/customerio_test.go
@@ -1,121 +1,98 @@
-//go:build detectors
-// +build detectors
-
package customerio
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestCustomerIO_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("CUSTOMERIO_TOKEN")
- inactiveSecret := testSecrets.MustGetField("CUSTOMERIO_INACTIVE")
- id := testSecrets.MustGetField("CUSTOMERIO_ID")
+var (
+ validPattern = `
+ # Configuration File: config.yaml
+ database:
+ host: $DB_HOST
+ port: $DB_PORT
+ username: $DB_USERNAME
+ password: $DB_PASS # IMPORTANT: Do not share this password publicly
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ api:
+ auth_type: "Basic"
+ in: "Header"
+ customerio_key: "bXQLU0kcl0A7kxCErc3L"
+ customerio_id: "tM2JFc8pmKHUmkdwhmgG"
+ base_url: "https://api.example.com/v1/user?access_key=$currencylayer_key"
+
+ # Notes:
+ # - Remember to rotate the secret every 90 days.
+ # - The above credentials should only be used in a secure environment.
+ `
+ secrets = []string{
+ "bXQLU0kcl0A7kxCErc3LbXQLU0kcl0A7kxCErc3L",
+ "bXQLU0kcl0A7kxCErc3LtM2JFc8pmKHUmkdwhmgG",
+ "tM2JFc8pmKHUmkdwhmgGbXQLU0kcl0A7kxCErc3L",
+ "tM2JFc8pmKHUmkdwhmgGtM2JFc8pmKHUmkdwhmgG",
}
+)
+
+func TestCustomerio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s ", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CustomerIO,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_CustomerIO,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: secrets,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("CustomerIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("CustomerIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}