Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Added pattern test cases for detectors starting with Alphabet C (#3564)
Kashif Khan6921ffda95b3
diff --git a/pkg/detectors/caflou/caflou_integration_test.go b/pkg/detectors/caflou/caflou_integration_test.gonew file mode 100644index 000000000..b8eb1bec3--- /dev/null+++ b/pkg/detectors/caflou/caflou_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package caflou++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCaflou_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CAFLOU")+ inactiveSecret := testSecrets.MustGetField("CAFLOU_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a caflou secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Caflou,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a caflou secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Caflou,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Caflou.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Caflou.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/caflou/caflou_test.go b/pkg/detectors/caflou/caflou_test.goindex b8eb1bec3..500a2fca0 100644--- a/pkg/detectors/caflou/caflou_test.go+++ b/pkg/detectors/caflou/caflou_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package caflouimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ base_url: "https://api.example.com/instances"+ api_key: $API_KEY+ caflou_auth_token: "Bearer b8SQoPKLMCBbwIm0XDzbZiDydUk9qNqqBlKnR5Nouwbjs9cv3D1azAXpiFq9WrfNlwxbCwDL2FWCheXmdYKZkMRZklahJh5NQZZY7Zf220hjGJOtKgFbWxy9xQ9hodQqsOOx9Of30qtTrnRxFPa9wxYkSBn"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "b8SQoPKLMCBbwIm0XDzbZiDydUk9qNqqBlKnR5Nouwbjs9cv3D1azAXpiFq9WrfNlwxbCwDL2FWCheXmdYKZkMRZklahJh5NQZZY7Zf220hjGJOtKgFbWxy9xQ9hodQqsOOx9Of30qtTrnRxFPa9wxYkSBn")-func TestCaflou_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CAFLOU")- inactiveSecret := testSecrets.MustGetField("CAFLOU_INACTIVE")+func TestCaflou_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a caflou secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Caflou,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a caflou secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Caflou,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Caflou.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Caflou.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/calendarific/calendarific_integration_test.go b/pkg/detectors/calendarific/calendarific_integration_test.gonew file mode 100644index 000000000..9b6fadba0--- /dev/null+++ b/pkg/detectors/calendarific/calendarific_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package calendarific++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCalendarific_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CALENDARIFIC_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CALENDARIFIC_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a calendarific secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Calendarific,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a calendarific secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Calendarific,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Calendarific.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Calendarific.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/calendarific/calendarific_test.go b/pkg/detectors/calendarific/calendarific_test.goindex 9b6fadba0..37f77a241 100644--- a/pkg/detectors/calendarific/calendarific_test.go+++ b/pkg/detectors/calendarific/calendarific_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package calendarificimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ calendarific_api_key: "M9qT0uymrS2FgJ78p9CpLIlFOBLUtmao"+ base_url: "https://api.calendarific.com/v1/holidays?api_key=$calendarific_api_key"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "M9qT0uymrS2FgJ78p9CpLIlFOBLUtmao")-func TestCalendarific_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CALENDARIFIC_TOKEN")- inactiveSecret := testSecrets.MustGetField("CALENDARIFIC_INACTIVE")+func TestCalendarific_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a calendarific secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Calendarific,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a calendarific secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Calendarific,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Calendarific.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Calendarific.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/calendlyapikey/calendlyapikey.go b/pkg/detectors/calendlyapikey/calendlyapikey.goindex 86c1da7a9..f63d96c87 100644--- a/pkg/detectors/calendlyapikey/calendlyapikey.go+++ b/pkg/detectors/calendlyapikey/calendlyapikey.go@@ -3,10 +3,11 @@ package calendlyapikeyimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/calendlyapikey/calendlyapikey_integration_test.go b/pkg/detectors/calendlyapikey/calendlyapikey_integration_test.gonew file mode 100644index 000000000..4f1e88a3a--- /dev/null+++ b/pkg/detectors/calendlyapikey/calendlyapikey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package calendlyapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCalendlyApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CALENDLYAPIKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CALENDLYAPIKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CalendlyApiKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CalendlyApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CalendlyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CalendlyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/calendlyapikey/calendlyapikey_test.go b/pkg/detectors/calendlyapikey/calendlyapikey_test.goindex 4f1e88a3a..943632729 100644--- a/pkg/detectors/calendlyapikey/calendlyapikey_test.go+++ b/pkg/detectors/calendlyapikey/calendlyapikey_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package calendlyapikeyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ api_key: $API_KEY+ base_url: "https://api.example.com/v1/user"+ calendly_auth_token: "Bearer eyJuL_8UF5AiQVfO2xlr_HaoSluHz9u-Q-s1qWDWvycQhR11J9wZTmYfFpKnawuIbKjA4i340DSpYI3d3E-oEZZdcHW4cLd_OASWu-H.eyJuOUikPwjw1RKXYXfcjjeqQwdWzA4uooei_ADIUX3of4UzwTjSaaEzLWMGopW4n9Fma0nINBD1qUp_OtbhuH6dmHyv94IeX-hUYla.A0rTdrx3sJ"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "eyJuL_8UF5AiQVfO2xlr_HaoSluHz9u-Q-s1qWDWvycQhR11J9wZTmYfFpKnawuIbKjA4i340DSpYI3d3E-oEZZdcHW4cLd_OASWu-H.eyJuOUikPwjw1RKXYXfcjjeqQwdWzA4uooei_ADIUX3of4UzwTjSaaEzLWMGopW4n9Fma0nINBD1qUp_OtbhuH6dmHyv94IeX-hUYla.A0rTdrx3sJ")-func TestCalendlyApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CALENDLYAPIKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("CALENDLYAPIKEY_INACTIVE")+func TestCalendlyAPIKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CalendlyApiKey,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a calendlyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CalendlyApiKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CalendlyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CalendlyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/calorieninja/calorieninja.go b/pkg/detectors/calorieninja/calorieninja.goindex 47f594080..362b55f10 100644--- a/pkg/detectors/calorieninja/calorieninja.go+++ b/pkg/detectors/calorieninja/calorieninja.go@@ -2,10 +2,11 @@ package calorieninjaimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/calorieninja/calorieninja_integration_test.go b/pkg/detectors/calorieninja/calorieninja_integration_test.gonew file mode 100644index 000000000..4b680cb75--- /dev/null+++ b/pkg/detectors/calorieninja/calorieninja_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package calorieninja++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCalorieninja_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CALORIENINJA")+ inactiveSecret := testSecrets.MustGetField("CALORIENINJA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CalorieNinja,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CalorieNinja,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Calorieninja.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Calorieninja.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/calorieninja/calorieninja_test.go b/pkg/detectors/calorieninja/calorieninja_test.goindex 4b680cb75..145c9a069 100644--- a/pkg/detectors/calorieninja/calorieninja_test.go+++ b/pkg/detectors/calorieninja/calorieninja_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package calorieninjaimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ calorieninja_api_key: "ix1aaifujilTcGEjB67e1EBBRXcr7r9cdChAR5hb"+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "ix1aaifujilTcGEjB67e1EBBRXcr7r9cdChAR5hb")-func TestCalorieninja_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CALORIENINJA")- inactiveSecret := testSecrets.MustGetField("CALORIENINJA_INACTIVE")+func TestCalorieNinja_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CalorieNinja,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a calorieninja secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CalorieNinja,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Calorieninja.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Calorieninja.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/campayn/campayn.go b/pkg/detectors/campayn/campayn.goindex c12aaee76..c39485ced 100644--- a/pkg/detectors/campayn/campayn.go+++ b/pkg/detectors/campayn/campayn.go@@ -2,10 +2,11 @@ package campaynimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/campayn/campayn_integration_test.go b/pkg/detectors/campayn/campayn_integration_test.gonew file mode 100644index 000000000..7cd07d212--- /dev/null+++ b/pkg/detectors/campayn/campayn_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package campayn++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCampayn_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CAMPAYN_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CAMPAYN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a campayn secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Campayn,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a campayn secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Campayn,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Campayn.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Campayn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/campayn/campayn_test.go b/pkg/detectors/campayn/campayn_test.goindex 7cd07d212..c5da83acc 100644--- a/pkg/detectors/campayn/campayn_test.go+++ b/pkg/detectors/campayn/campayn_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package campaynimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCampayn_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CAMPAYN_TOKEN")- inactiveSecret := testSecrets.MustGetField("CAMPAYN_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ campayn_api_key: "z6q8z47eu46wri18ygu6pc68vsizprt83jrlu5ustliyhktzoxbzhf1ycdaka978"+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "z6q8z47eu46wri18ygu6pc68vsizprt83jrlu5ustliyhktzoxbzhf1ycdaka978"+)++func TestCampayn_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a campayn secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Campayn,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a campayn secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Campayn,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Campayn.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Campayn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cannyio/cannyio_integration_test.go b/pkg/detectors/cannyio/cannyio_integration_test.gonew file mode 100644index 000000000..7226dcaf8--- /dev/null+++ b/pkg/detectors/cannyio/cannyio_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cannyio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCannyIo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CANNYIO_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CANNYIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cannyio secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CannyIo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cannyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CannyIo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CannyIo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CannyIo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cannyio/cannyio_test.go b/pkg/detectors/cannyio/cannyio_test.goindex 7226dcaf8..d4dba11b6 100644--- a/pkg/detectors/cannyio/cannyio_test.go+++ b/pkg/detectors/cannyio/cannyio_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package cannyioimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ cannyio_api_key: "faiiahli-goke-db0r-oxli-s20dgab9a0iv"+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "faiiahli-goke-db0r-oxli-s20dgab9a0iv")-func TestCannyIo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CANNYIO_TOKEN")- inactiveSecret := testSecrets.MustGetField("CANNYIO_INACTIVE")+func TestCannyio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cannyio secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CannyIo,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cannyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CannyIo,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CannyIo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CannyIo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/capsulecrm/capsulecrm.go b/pkg/detectors/capsulecrm/capsulecrm.goindex f915e6932..46f34b157 100644--- a/pkg/detectors/capsulecrm/capsulecrm.go+++ b/pkg/detectors/capsulecrm/capsulecrm.go@@ -3,10 +3,11 @@ package capsulecrmimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/capsulecrm/capsulecrm_integration_test.go b/pkg/detectors/capsulecrm/capsulecrm_integration_test.gonew file mode 100644index 000000000..c63a12412--- /dev/null+++ b/pkg/detectors/capsulecrm/capsulecrm_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package capsulecrm++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCapsuleCRM_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CAPSULECRM")+ inactiveSecret := testSecrets.MustGetField("CAPSULECRM_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CapsuleCRM,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CapsuleCRM,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CapsuleCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CapsuleCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/capsulecrm/capsulecrm_test.go b/pkg/detectors/capsulecrm/capsulecrm_test.goindex c63a12412..18f502220 100644--- a/pkg/detectors/capsulecrm/capsulecrm_test.go+++ b/pkg/detectors/capsulecrm/capsulecrm_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package capsulecrmimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ api_key: ""+ base_url: "https://api.example.com/v1/user"+ capsulecrm_auth_token: "Bearer ULeIqU-4ss+YImZYsyjPLSsm.9H.SZJ1v.KxT1D-zbaW6sg5b0R5g=koBH4X62hC"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "ULeIqU-4ss+YImZYsyjPLSsm.9H.SZJ1v.KxT1D-zbaW6sg5b0R5g=koBH4X62hC")-func TestCapsuleCRM_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CAPSULECRM")- inactiveSecret := testSecrets.MustGetField("CAPSULECRM_INACTIVE")+func TestCapsulecrm_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CapsuleCRM,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a capsulecrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CapsuleCRM,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CapsuleCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CapsuleCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/captaindata/v1/captaindata_integration_test.go b/pkg/detectors/captaindata/v1/captaindata_integration_test.gonew file mode 100644index 000000000..c77264ade--- /dev/null+++ b/pkg/detectors/captaindata/v1/captaindata_integration_test.go@@ -0,0 +1,118 @@+//go:build detectors+// +build detectors++package captaindata++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCaptainData_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ projId := testSecrets.MustGetField("CAPTAINDATA_PROJID")+ secret := testSecrets.MustGetField("CAPTAINDATA")+ inactiveSecret := testSecrets.MustGetField("CAPTAINDATA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within", projId, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CaptainData,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within but not valid", projId, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CaptainData,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CaptainData.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CaptainData.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/captaindata/v1/captaindata_test.go b/pkg/detectors/captaindata/v1/captaindata_test.goindex c77264ade..71bc022a8 100644--- a/pkg/detectors/captaindata/v1/captaindata_test.go+++ b/pkg/detectors/captaindata/v1/captaindata_test.go@@ -1,117 +1,83 @@-//go:build detectors-// +build detectors-package captaindataimport ("context"- "fmt""testing"- "time"-- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCaptainData_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- projId := testSecrets.MustGetField("CAPTAINDATA_PROJID")- secret := testSecrets.MustGetField("CAPTAINDATA")- inactiveSecret := testSecrets.MustGetField("CAPTAINDATA_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestCaptainData_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within", projId, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CaptainData,- Verified: true,- },- },- wantErr: false,+ name: "typical pattern",+ input: "captaindata_project = '12345678-1234-1234-1234-123456789012' captaindata_api_key = '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'",+ want: []string{"12345678-1234-1234-1234-1234567890121234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a captaindata project %s with captaindata secret %s within but not valid", projId, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CaptainData,- Verified: false,- },+ name: "finds all matches",+ input: `captaindata_project1 = '12345678-1234-1234-1234-123456789012' captaindata_api_key1 = '1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef'+captaindata_project2 = '87654321-4321-4321-4321-210987654321' captaindata_api_key2 = 'fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321'`,+ want: []string{+ "12345678-1234-1234-1234-1234567890121234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",+ "12345678-1234-1234-1234-123456789012fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321",+ "87654321-4321-4321-4321-210987654321fedcba0987654321fedcba0987654321fedcba0987654321fedcba0987654321",+ "87654321-4321-4321-4321-2109876543211234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef",},- wantErr: false,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: "captaindata_project = '123456' captaindata_api_key = '1234567890'",+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CaptainData.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}- got[i].Raw = nil}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CaptainData.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/carboninterface/carboninterface.go b/pkg/detectors/carboninterface/carboninterface.goindex d934c998f..977c86549 100644--- a/pkg/detectors/carboninterface/carboninterface.go+++ b/pkg/detectors/carboninterface/carboninterface.go@@ -3,10 +3,11 @@ package carboninterfaceimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/carboninterface/carboninterface_integration_test.go b/pkg/detectors/carboninterface/carboninterface_integration_test.gonew file mode 100644index 000000000..e002caa1c--- /dev/null+++ b/pkg/detectors/carboninterface/carboninterface_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package carboninterface++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCarbonInterface_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CARBONINTERFACE")+ inactiveSecret := testSecrets.MustGetField("CARBONINTERFACE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CarbonInterface,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CarbonInterface,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CarbonInterface.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CarbonInterface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/carboninterface/carboninterface_test.go b/pkg/detectors/carboninterface/carboninterface_test.goindex e002caa1c..ab1b0fe9b 100644--- a/pkg/detectors/carboninterface/carboninterface_test.go+++ b/pkg/detectors/carboninterface/carboninterface_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package carboninterfaceimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ api_key: ""+ base_url: "https://api.example.com/v1/user"+ carboninterface_auth_token: "Bearer PkN3gWaSHSIjz188TjD4h"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "PkN3gWaSHSIjz188TjD4h")-func TestCarbonInterface_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CARBONINTERFACE")- inactiveSecret := testSecrets.MustGetField("CARBONINTERFACE_INACTIVE")+func TestCarbonInterface_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CarbonInterface,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a carboninterface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CarbonInterface,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CarbonInterface.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CarbonInterface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cashboard/cashboard.go b/pkg/detectors/cashboard/cashboard.goindex 65efbee34..364a7d749 100644--- a/pkg/detectors/cashboard/cashboard.go+++ b/pkg/detectors/cashboard/cashboard.go@@ -4,16 +4,17 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/cashboard/cashboard_integration_test.go b/pkg/detectors/cashboard/cashboard_integration_test.gonew file mode 100644index 000000000..4b4fd5db0--- /dev/null+++ b/pkg/detectors/cashboard/cashboard_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package cashboard++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCashboard_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CASHBOARD")+ user := testSecrets.MustGetField("SCANNER_USERNAME")+ inactiveSecret := testSecrets.MustGetField("CASHBOARD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s", secret, user)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cashboard,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cashboard,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cashboard.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cashboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cashboard/cashboard_test.go b/pkg/detectors/cashboard/cashboard_test.goindex 4b4fd5db0..e6c20e607 100644--- a/pkg/detectors/cashboard/cashboard_test.go+++ b/pkg/detectors/cashboard/cashboard_test.go@@ -1,121 +1,93 @@-//go:build detectors-// +build detectors-package cashboardimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCashboard_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CASHBOARD")- user := testSecrets.MustGetField("SCANNER_USERNAME")- inactiveSecret := testSecrets.MustGetField("CASHBOARD_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ cashboard_key: "F1A-NEI-HY4-PZK"+ cashboard_user: "ts7z"+ auth_type: Basic+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "F1A-NEI-HY4-PZKts7z"+)++func TestCashBoard_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s", secret, user)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cashboard,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cashboard secret %s within %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cashboard,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cashboard.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cashboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/caspio/caspio_integration_test.go b/pkg/detectors/caspio/caspio_integration_test.gonew file mode 100644index 000000000..b285d94cf--- /dev/null+++ b/pkg/detectors/caspio/caspio_integration_test.go@@ -0,0 +1,119 @@+//go:build detectors+// +build detectors++package caspio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCaspio_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CASPIO")+ id := testSecrets.MustGetField("CASPIO_ID")+ subdomain := testSecrets.MustGetField("CASPIO_SUBDOMAIN")+ inactiveSecret := testSecrets.MustGetField("CASPIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s", secret, id, subdomain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Caspio,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s but not valid", inactiveSecret, id, subdomain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Caspio,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Caspio.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Caspio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/caspio/caspio_test.go b/pkg/detectors/caspio/caspio_test.goindex b285d94cf..7d56a91b8 100644--- a/pkg/detectors/caspio/caspio_test.go+++ b/pkg/detectors/caspio/caspio_test.go@@ -1,118 +1,98 @@-//go:build detectors-// +build detectors-package caspioimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCaspio_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CASPIO")- id := testSecrets.MustGetField("CASPIO_ID")- subdomain := testSecrets.MustGetField("CASPIO_SUBDOMAIN")- inactiveSecret := testSecrets.MustGetField("CASPIO_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ caspio_id: "qye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01asty"+ caspio_secret: "x5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibe"+ caspio_domain: xlo0xo2s+ auth_type: Client Credentials+ base_url: "https://$caspio_domain.caspio.com/v1/user"+ auth_token: ""- type args struct {- ctx context.Context- data []byte- verify bool+ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secrets = []string{+ "qye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyqye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyxlo0xo2s",+ "qye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyx5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibexlo0xo2s",+ "x5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibeqye6c979b55w55hz7nqrdgax3pufdsigwvhofsrxalgk01astyxlo0xo2s",+ "x5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibex5vzi1o2fmnjywilv3obwx5n041f56v54lgral6znyccet8ibexlo0xo2s",}+)++func TestCaspio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s", secret, id, subdomain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Caspio,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a caspio secret %s within caspio %s and caspio subdomain %s but not valid", inactiveSecret, id, subdomain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Caspio,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: secrets,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Caspio.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Caspio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/censys/censys.go b/pkg/detectors/censys/censys.goindex 485c12c64..43f8f11c6 100644--- a/pkg/detectors/censys/censys.go+++ b/pkg/detectors/censys/censys.go@@ -2,10 +2,11 @@ package censysimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/censys/censys_integration_test.go b/pkg/detectors/censys/censys_integration_test.gonew file mode 100644index 000000000..1b20923c4--- /dev/null+++ b/pkg/detectors/censys/censys_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package censys++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCensys_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CENSYS")+ id := testSecrets.MustGetField("CENSYS_ID")+ inactiveSecret := testSecrets.MustGetField("CENSYS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Censys,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Censys,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Censys.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Censys.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/censys/censys_test.go b/pkg/detectors/censys/censys_test.goindex 1b20923c4..697004fef 100644--- a/pkg/detectors/censys/censys_test.go+++ b/pkg/detectors/censys/censys_test.go@@ -1,121 +1,93 @@-//go:build detectors-// +build detectors-package censysimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ censys_key: "roLtT01S6znCRhgoSwNNKY8O7AELn8e4"+ censys_user: "p4cuaz9fuonwmbfkc4di5uqsizp4yyttpu-q"+ auth_type: Basic+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "roLtT01S6znCRhgoSwNNKY8O7AELn8e4p4cuaz9fuonwmbfkc4di5uqsizp4yyttpu-q")-func TestCensys_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CENSYS")- id := testSecrets.MustGetField("CENSYS_ID")- inactiveSecret := testSecrets.MustGetField("CENSYS_INACTIVE")+func TestCensys_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Censys,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a censys secret %s within censys %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Censys,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Censys.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Censys.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/centralstationcrm/centralstationcrm.go b/pkg/detectors/centralstationcrm/centralstationcrm.goindex 2d20d6630..a32a7f953 100644--- a/pkg/detectors/centralstationcrm/centralstationcrm.go+++ b/pkg/detectors/centralstationcrm/centralstationcrm.go@@ -2,10 +2,11 @@ package centralstationcrmimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/centralstationcrm/centralstationcrm_integration_test.go b/pkg/detectors/centralstationcrm/centralstationcrm_integration_test.gonew file mode 100644index 000000000..364ed8e82--- /dev/null+++ b/pkg/detectors/centralstationcrm/centralstationcrm_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package centralstationcrm++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCentralStationCRM_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CENTRALSTATIONCRM")+ inactiveSecret := testSecrets.MustGetField("CENTRALSTATIONCRM_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CentralStationCRM,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CentralStationCRM,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CentralStationCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CentralStationCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/centralstationcrm/centralstationcrm_test.go b/pkg/detectors/centralstationcrm/centralstationcrm_test.goindex 364ed8e82..01fe54b65 100644--- a/pkg/detectors/centralstationcrm/centralstationcrm_test.go+++ b/pkg/detectors/centralstationcrm/centralstationcrm_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package centralstationcrmimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ centralstationcrm_api_key: "gyeyy7soy4lxx7yenw56iba4szfu1f"+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "gyeyy7soy4lxx7yenw56iba4szfu1f")-func TestCentralStationCRM_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CENTRALSTATIONCRM")- inactiveSecret := testSecrets.MustGetField("CENTRALSTATIONCRM_INACTIVE")+func TestCentralStationCRM_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CentralStationCRM,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a centralstationcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CentralStationCRM,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CentralStationCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CentralStationCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cexio/cexio.go b/pkg/detectors/cexio/cexio.goindex 56b06c020..3311240a4 100644--- a/pkg/detectors/cexio/cexio.go+++ b/pkg/detectors/cexio/cexio.go@@ -6,7 +6,6 @@ import ("crypto/sha256""encoding/hex""encoding/json"- regexp "github.com/wasilibs/go-re2""io""net/http""net/url"@@ -14,12 +13,14 @@ import ("strings""time"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/cexio/cexio_integration_test.go b/pkg/detectors/cexio/cexio_integration_test.gonew file mode 100644index 000000000..0fd5b1422--- /dev/null+++ b/pkg/detectors/cexio/cexio_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cexio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCexIO_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ userId := testSecrets.MustGetField("CEXIO_USERID")+ key := testSecrets.MustGetField("CEXIO_KEY")+ inactiveKey := testSecrets.MustGetField("CEXIO_KEY_INACTIVE")+ secret := testSecrets.MustGetField("CEXIO")+ inactiveSecret := testSecrets.MustGetField("CEXIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within", userId, key, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CexIO,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within but not valid", userId, inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CexIO,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CexIO.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CexIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/cexio/cexio_test.go b/pkg/detectors/cexio/cexio_test.goindex 0fd5b1422..36e8ced85 100644--- a/pkg/detectors/cexio/cexio_test.go+++ b/pkg/detectors/cexio/cexio_test.go@@ -1,119 +1,98 @@-//go:build detectors-// +build detectors-package cexioimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCexIO_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- userId := testSecrets.MustGetField("CEXIO_USERID")- key := testSecrets.MustGetField("CEXIO_KEY")- inactiveKey := testSecrets.MustGetField("CEXIO_KEY_INACTIVE")- secret := testSecrets.MustGetField("CEXIO")- inactiveSecret := testSecrets.MustGetField("CEXIO_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ cexio_key: "bVI24QF2B8omVr9ddfxSHtkb18D"+ cexio_secret: "2m2pEr2OLi48y2NCpSbPVwJqb"+ cex.io_userID: "zd930167221"+ auth_type: Signature+ base_url: "https://api.example.com/v1/user"+ auth_token: ""- type args struct {- ctx context.Context- data []byte- verify bool+ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secrets = []string{+ "zd9301672212m2pEr2OLi48y2NCpSbPVwJqb",+ "zd9301672212m2pEr2OLi48y2NCpSbPVwJqb",+ "zd930167221bVI24QF2B8omVr9ddfxSHtkb18D",+ "zd930167221bVI24QF2B8omVr9ddfxSHtkb18D",}+)++func TestCexio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within", userId, key, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CexIO,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cexio userId %s with cexio key %s and cexio secret %s within but not valid", userId, inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CexIO,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: secrets,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CexIO.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CexIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/chartmogul/chartmogul.go b/pkg/detectors/chartmogul/chartmogul.goindex 5af07ec49..689a54b33 100644--- a/pkg/detectors/chartmogul/chartmogul.go+++ b/pkg/detectors/chartmogul/chartmogul.go@@ -4,10 +4,11 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/chartmogul/chartmogul_integration_test.go b/pkg/detectors/chartmogul/chartmogul_integration_test.gonew file mode 100644index 000000000..31d43fb47--- /dev/null+++ b/pkg/detectors/chartmogul/chartmogul_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package chartmogul++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestChartmogul_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CHARTMOGUL")+ inactiveSecret := testSecrets.MustGetField("CHARTMOGUL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Chartmogul,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Chartmogul,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Chartmogul.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Chartmogul.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/chartmogul/chartmogul_test.go b/pkg/detectors/chartmogul/chartmogul_test.goindex 31d43fb47..ac16af0b8 100644--- a/pkg/detectors/chartmogul/chartmogul_test.go+++ b/pkg/detectors/chartmogul/chartmogul_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package chartmogulimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ chartmogul_key: "e8hwspf91879g0u267yq1bkoxquvwndk"+ auth_type: Basic+ base_url: "https://api.example.com/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "e8hwspf91879g0u267yq1bkoxquvwndk")-func TestChartmogul_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CHARTMOGUL")- inactiveSecret := testSecrets.MustGetField("CHARTMOGUL_INACTIVE")+func TestChartMogul_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Chartmogul,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a chartmogul secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Chartmogul,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Chartmogul.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Chartmogul.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/chatbot/chatbot_integration_test.go b/pkg/detectors/chatbot/chatbot_integration_test.gonew file mode 100644index 000000000..ff62f3997--- /dev/null+++ b/pkg/detectors/chatbot/chatbot_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package chatbot++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestChatbot_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CHATBOT_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CHATBOT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a chatbot secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Chatbot,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a chatbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Chatbot,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Chatbot.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Chatbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/chatbot/chatbot_test.go b/pkg/detectors/chatbot/chatbot_test.goindex ff62f3997..e7f3fff44 100644--- a/pkg/detectors/chatbot/chatbot_test.go+++ b/pkg/detectors/chatbot/chatbot_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package chatbotimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: Bearer+ base_url: "https://api.example.com/v1/user"+ chatbot_auth_token: "Bearer 5RzDGrpFKkrA_90yM_BFmyxKKAQkgu0B"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "5RzDGrpFKkrA_90yM_BFmyxKKAQkgu0B")-func TestChatbot_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CHATBOT_TOKEN")- inactiveSecret := testSecrets.MustGetField("CHATBOT_INACTIVE")+func TestChatBot_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a chatbot secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Chatbot,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a chatbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Chatbot,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Chatbot.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Chatbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/chatfule/chatfule.go b/pkg/detectors/chatfule/chatfule.goindex d947ed7b7..067ca140a 100644--- a/pkg/detectors/chatfule/chatfule.go+++ b/pkg/detectors/chatfule/chatfule.go@@ -3,10 +3,11 @@ package chatfuleimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/chatfule/chatfule_integration_test.go b/pkg/detectors/chatfule/chatfule_integration_test.gonew file mode 100644index 000000000..405174ed6--- /dev/null+++ b/pkg/detectors/chatfule/chatfule_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package chatfule++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestChatfule_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CHATFULE")+ inactiveSecret := testSecrets.MustGetField("CHATFULE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Chatfule,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Chatfule,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Chatfuel.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Chatfuel.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/chatfule/chatfule_test.go b/pkg/detectors/chatfule/chatfule_test.goindex 405174ed6..0f98714ca 100644--- a/pkg/detectors/chatfule/chatfule_test.go+++ b/pkg/detectors/chatfule/chatfule_test.go@@ -1,116 +1,90 @@-//go:build detectors-// +build detectors-package chatfuleimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestChatfule_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CHATFULE")- inactiveSecret := testSecrets.MustGetField("CHATFULE_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: Bearer+ base_url: "https://api.example.com/v1/user"+ chatfuel_auth_token: "Bearer 22ZEyoBNMXpT6rHmbuBlIJ8n19vo3tHzNTQDUku00WhHuBCAlkfkn8kQkXslseKEHARZthTrm8QfErQ5auXEr8teFIt6stHYi9sfJXM7IK0vEsezKFQwADCvMhX202eL"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "22ZEyoBNMXpT6rHmbuBlIJ8n19vo3tHzNTQDUku00WhHuBCAlkfkn8kQkXslseKEHARZthTrm8QfErQ5auXEr8teFIt6stHYi9sfJXM7IK0vEsezKFQwADCvMhX202eL"+)++func TestChatFule_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Chatfule,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a chatfuel secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Chatfule,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Chatfuel.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Chatfuel.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/checio/checio.go b/pkg/detectors/checio/checio.goindex c7a58d91d..5563c70b2 100644--- a/pkg/detectors/checio/checio.go+++ b/pkg/detectors/checio/checio.go@@ -2,11 +2,11 @@ package checioimport ("context"- // "log"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/checio/checio_integration_test.go b/pkg/detectors/checio/checio_integration_test.gonew file mode 100644index 000000000..e2fd6eafa--- /dev/null+++ b/pkg/detectors/checio/checio_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package checio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestChecIO_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CHECIO")+ inactiveSecret := testSecrets.MustGetField("CHECIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a checio secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ChecIO,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a checio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ChecIO,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ChecIO.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ChecIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/checio/checio_test.go b/pkg/detectors/checio/checio_test.goindex e2fd6eafa..ae7a224bf 100644--- a/pkg/detectors/checio/checio_test.go+++ b/pkg/detectors/checio/checio_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package checioimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: N/A+ base_url: "https://api.example.com/v1/user"+ checio_auth_token: "X-Authorization pk_k64v4e7f5vfun5efk7kscnvuwiuo9ioxbvxjq8qrdga0p"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "pk_k64v4e7f5vfun5efk7kscnvuwiuo9ioxbvxjq8qrdga0p")-func TestChecIO_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CHECIO")- inactiveSecret := testSecrets.MustGetField("CHECIO_INACTIVE")+func TestChecio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a checio secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ChecIO,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a checio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ChecIO,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ChecIO.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ChecIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/checklyhq/checklyhq.go b/pkg/detectors/checklyhq/checklyhq.goindex 274574b1c..607a331d2 100644--- a/pkg/detectors/checklyhq/checklyhq.go+++ b/pkg/detectors/checklyhq/checklyhq.go@@ -3,10 +3,11 @@ package checklyhqimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/checklyhq/checklyhq_integration_test.go b/pkg/detectors/checklyhq/checklyhq_integration_test.gonew file mode 100644index 000000000..8985924f7--- /dev/null+++ b/pkg/detectors/checklyhq/checklyhq_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package checklyhq++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestChecklyHQ_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CHECKLYHQ")+ inactiveSecret := testSecrets.MustGetField("CHECKLYHQ_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ChecklyHQ,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ChecklyHQ,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ChecklyHQ.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ChecklyHQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/checklyhq/checklyhq_test.go b/pkg/detectors/checklyhq/checklyhq_test.goindex 8985924f7..ff1c031a0 100644--- a/pkg/detectors/checklyhq/checklyhq_test.go+++ b/pkg/detectors/checklyhq/checklyhq_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package checklyhqimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: Bearer+ base_url: "https://api.example.com/v1/user"+ checklyhq_auth_token: "Bearer r3sd5apfe7p3eg1318qpbtxo36gwcct2"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "r3sd5apfe7p3eg1318qpbtxo36gwcct2")-func TestChecklyHQ_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CHECKLYHQ")- inactiveSecret := testSecrets.MustGetField("CHECKLYHQ_INACTIVE")+func TestChecklyhq_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ChecklyHQ,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a checklyhq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ChecklyHQ,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ChecklyHQ.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ChecklyHQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/checkout/checkout.go b/pkg/detectors/checkout/checkout.goindex c14dcb129..def4425b7 100644--- a/pkg/detectors/checkout/checkout.go+++ b/pkg/detectors/checkout/checkout.go@@ -2,10 +2,11 @@ package checkoutimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/checkout/checkout_integration_test.go b/pkg/detectors/checkout/checkout_integration_test.gonew file mode 100644index 000000000..0337cfe5b--- /dev/null+++ b/pkg/detectors/checkout/checkout_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package checkout++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCheckout_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CHECKOUT")+ inactiveSecret := testSecrets.MustGetField("CHECKOUT_INACTIVE")+ secretId := testSecrets.MustGetField("CHECKOUT_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within", secret, secretId)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Checkout,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within but not valid", inactiveSecret, secretId)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Checkout,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Checkout.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Checkout.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/checkout/checkout_test.go b/pkg/detectors/checkout/checkout_test.goindex 0337cfe5b..7132cb6f2 100644--- a/pkg/detectors/checkout/checkout_test.go+++ b/pkg/detectors/checkout/checkout_test.go@@ -1,121 +1,91 @@-//go:build detectors-// +build detectors-package checkoutimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: API-Key+ base_url: "https://api.checkout.com/v1/customers/cus_DaZoK0ioakAfFaj6fyqSFQZatk"+ checkout_api_key: "sk_test_14a67eEd-21Fd-B18d-2B8D-275697febE7D"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "sk_test_14a67eEd-21Fd-B18d-2B8D-275697febE7Dcus_DaZoK0ioakAfFaj6fyqSFQZatk")-func TestCheckout_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CHECKOUT")- inactiveSecret := testSecrets.MustGetField("CHECKOUT_INACTIVE")- secretId := testSecrets.MustGetField("CHECKOUT_ID")+func TestCheckout_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within", secret, secretId)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Checkout,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a checkout secret %s with checkout id %s within but not valid", inactiveSecret, secretId)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Checkout,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Checkout.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Checkout.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cicero/cicero.go b/pkg/detectors/cicero/cicero.goindex 6014f49a8..3d64ae941 100644--- a/pkg/detectors/cicero/cicero.go+++ b/pkg/detectors/cicero/cicero.go@@ -3,10 +3,11 @@ package ciceroimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cicero/cicero_integration_test.go b/pkg/detectors/cicero/cicero_integration_test.gonew file mode 100644index 000000000..7bb562b69--- /dev/null+++ b/pkg/detectors/cicero/cicero_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cicero++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCicero_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CICERO")+ inactiveSecret := testSecrets.MustGetField("CICERO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cicero secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cicero,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cicero secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cicero,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cicero.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cicero.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cicero/cicero_test.go b/pkg/detectors/cicero/cicero_test.goindex 7bb562b69..617c7fa3c 100644--- a/pkg/detectors/cicero/cicero_test.go+++ b/pkg/detectors/cicero/cicero_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package ciceroimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ base_url: "https://api.cicero.com/v1/user?key=sbxod2yo56quitbyujhkig3mgtu6z49f4hh56va6"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "sbxod2yo56quitbyujhkig3mgtu6z49f4hh56va6")-func TestCicero_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CICERO")- inactiveSecret := testSecrets.MustGetField("CICERO_INACTIVE")+func TestCicero_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cicero secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cicero,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cicero secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cicero,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cicero.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cicero.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/circleci/circleci.go b/pkg/detectors/circleci/circleci.goindex 6c04002ac..b81745bdf 100644--- a/pkg/detectors/circleci/circleci.go+++ b/pkg/detectors/circleci/circleci.go@@ -2,9 +2,10 @@ package circleciimport ("context"- regexp "github.com/wasilibs/go-re2""net/http"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/circleci/circleci_integration_test.go b/pkg/detectors/circleci/circleci_integration_test.gonew file mode 100644index 000000000..34d685903--- /dev/null+++ b/pkg/detectors/circleci/circleci_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package circleci++import (+ "context"+ "fmt"+ "os"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCircleCI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CIRCLECI")+ secretInactive := testSecrets.MustGetField("CIRCLECI_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Circle,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a circle secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Circle,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CircleCI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CircleCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/circleci/circleci_test.go b/pkg/detectors/circleci/circleci_test.goindex 34d685903..1730c18f5 100644--- a/pkg/detectors/circleci/circleci_test.go+++ b/pkg/detectors/circleci/circleci_test.go@@ -1,122 +1,91 @@-//go:build detectors-// +build detectors-package circleciimport ("context"- "fmt"- "os""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCircleCI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CIRCLECI")- secretInactive := testSecrets.MustGetField("CIRCLECI_INACTIVE")- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: API-Key+ base_url: "https://api.example.com/v1/user"+ circleci_api_key: "4a4fFEA0Cb7760ee42Bb1Dc82b1E4E5eDCacB9E7"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "4a4fFEA0Cb7760ee42Bb1Dc82b1E4E5eDCacB9E7"+)++func TestCircleCI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a circle secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Circle,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a circle secret %s within", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Circle,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CircleCI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- if os.Getenv("FORCE_PASS_DIFF") == "true" {++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CircleCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clarifai/clarifai.go b/pkg/detectors/clarifai/clarifai.goindex 29018e406..3dce5439c 100644--- a/pkg/detectors/clarifai/clarifai.go+++ b/pkg/detectors/clarifai/clarifai.go@@ -3,10 +3,11 @@ package clarifaiimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clarifai/clarifai_integration_test.go b/pkg/detectors/clarifai/clarifai_integration_test.gonew file mode 100644index 000000000..96ab6fe5a--- /dev/null+++ b/pkg/detectors/clarifai/clarifai_integration_test.go@@ -0,0 +1,137 @@+//go:build detectors+// +build detectors++package clarifai++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClarifai_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ apiKey := testSecrets.MustGetField("CLARIFAI_API_KEY")+ inactiveApiKey := testSecrets.MustGetField("CLARIFAI_API_KEY_INACTIVE")+ pat := testSecrets.MustGetField("CLARIFAI_PAT")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clarifai api key %s within", apiKey)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clarifai,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clarifai pat %s within", pat)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clarifai,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clarifai secret %s within but unverified", inactiveApiKey)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clarifai,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Clarifai.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Clarifai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clarifai/clarifai_test.go b/pkg/detectors/clarifai/clarifai_test.goindex 96ab6fe5a..62d226964 100644--- a/pkg/detectors/clarifai/clarifai_test.go+++ b/pkg/detectors/clarifai/clarifai_test.go@@ -1,137 +1,91 @@-//go:build detectors-// +build detectors-package clarifaiimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ api:+ auth_type: API-Key+ base_url: "https://api.example.com/v1/user"+ clarifai_key: "WCFcfUCsl2P3vCJuFgDuLeUXduycoZli"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "WCFcfUCsl2P3vCJuFgDuLeUXduycoZli")-func TestClarifai_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- apiKey := testSecrets.MustGetField("CLARIFAI_API_KEY")- inactiveApiKey := testSecrets.MustGetField("CLARIFAI_API_KEY_INACTIVE")- pat := testSecrets.MustGetField("CLARIFAI_PAT")+func TestClarifai_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clarifai api key %s within", apiKey)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clarifai,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clarifai pat %s within", pat)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clarifai,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clarifai secret %s within but unverified", inactiveApiKey)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clarifai,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Clarifai.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Clarifai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clearbit/clearbit.go b/pkg/detectors/clearbit/clearbit.goindex ef56a93ab..b45a2b28a 100644--- a/pkg/detectors/clearbit/clearbit.go+++ b/pkg/detectors/clearbit/clearbit.go@@ -3,10 +3,11 @@ package clearbitimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clearbit/clearbit_integration_test.go b/pkg/detectors/clearbit/clearbit_integration_test.gonew file mode 100644index 000000000..da8ba1c26--- /dev/null+++ b/pkg/detectors/clearbit/clearbit_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package clearbit++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClearbit_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLEARBIT")+ inactiveSecret := testSecrets.MustGetField("CLEARBIT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clearbit secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clearbit,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clearbit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clearbit,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Clearbit.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Clearbit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clearbit/clearbit_test.go b/pkg/detectors/clearbit/clearbit_test.goindex da8ba1c26..522d69119 100644--- a/pkg/detectors/clearbit/clearbit_test.go+++ b/pkg/detectors/clearbit/clearbit_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package clearbitimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: Bearer+ base_url: "https://api.example.com/v1/user"+ clearbit_auth_token: "Bearer 9itvicgfiyq3ry6g03qwhwc_0s309dy8woh"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "9itvicgfiyq3ry6g03qwhwc_0s309dy8woh")-func TestClearbit_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLEARBIT")- inactiveSecret := testSecrets.MustGetField("CLEARBIT_INACTIVE")+func TestClearBit_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clearbit secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clearbit,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clearbit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clearbit,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Clearbit.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Clearbit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clickhelp/clickhelp.go b/pkg/detectors/clickhelp/clickhelp.goindex 3d2f839f6..f258f8dd8 100644--- a/pkg/detectors/clickhelp/clickhelp.go+++ b/pkg/detectors/clickhelp/clickhelp.go@@ -4,16 +4,17 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/clickhelp/clickhelp_integration_test.go b/pkg/detectors/clickhelp/clickhelp_integration_test.gonew file mode 100644index 000000000..16965f63d--- /dev/null+++ b/pkg/detectors/clickhelp/clickhelp_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package clickhelp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClickhelp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ email := testSecrets.MustGetField("SCANNERS_EMAIL")+ server := testSecrets.MustGetField("CLICKHELP_SERVER")+ key := testSecrets.MustGetField("CLICKHELP")+ inactiveKey := testSecrets.MustGetField("CLICKHELP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s", key, email, server)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClickHelp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s but not valid", inactiveKey, email, server)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClickHelp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Clickhelp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Clickhelp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clickhelp/clickhelp_test.go b/pkg/detectors/clickhelp/clickhelp_test.goindex 16965f63d..f8ce474a0 100644--- a/pkg/detectors/clickhelp/clickhelp_test.go+++ b/pkg/detectors/clickhelp/clickhelp_test.go@@ -1,122 +1,97 @@-//go:build detectors-// +build detectors-package clickhelpimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestClickhelp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- email := testSecrets.MustGetField("SCANNERS_EMAIL")- server := testSecrets.MustGetField("CLICKHELP_SERVER")- key := testSecrets.MustGetField("CLICKHELP")- inactiveKey := testSecrets.MustGetField("CLICKHELP_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly- type args struct {- ctx context.Context- data []byte- verify bool+ api:+ user_email: "[email protected]"+ clickhelp_key: "XzUkp562BtmjfRGoOGBiLLNu"+ clickhelp_domain: testingdev.try.clickhelp.co+ auth_type: Basic+ base_url: "https://testing-dev.try.clickhelp.co/v1/user"+ auth_token: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secrets = []string{+ "[email protected]",+ "[email protected]",}+)++func TestClickHelp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s", key, email, server)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClickHelp,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clickhelp secret %s within clickhelp email %s and clickhelp server %s but not valid", inactiveKey, email, server)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClickHelp,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: secrets,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Clickhelp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Clickhelp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clicksendsms/clicksendsms.go b/pkg/detectors/clicksendsms/clicksendsms.goindex f036f39e4..67bd60fed 100644--- a/pkg/detectors/clicksendsms/clicksendsms.go+++ b/pkg/detectors/clicksendsms/clicksendsms.go@@ -4,16 +4,17 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/clicksendsms/clicksendsms_integration_test.go b/pkg/detectors/clicksendsms/clicksendsms_integration_test.gonew file mode 100644index 000000000..9df9c4c63--- /dev/null+++ b/pkg/detectors/clicksendsms/clicksendsms_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package clicksendsms++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClickSendsms_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLICKSENDSMS_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CLICKSENDSMS_INACTIVE")+ email := testSecrets.MustGetField("CLICKSENDSMS_EMAIL")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s", secret, email)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClickSendsms,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s but not valid", inactiveSecret, email)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClickSendsms,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ClickSendsms.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ClickSendsms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clicksendsms/clicksendsms_test.go b/pkg/detectors/clicksendsms/clicksendsms_test.goindex 9df9c4c63..4bdc3dbca 100644--- a/pkg/detectors/clicksendsms/clicksendsms_test.go+++ b/pkg/detectors/clicksendsms/clicksendsms_test.go@@ -1,121 +1,92 @@-//go:build detectors-// +build detectors-package clicksendsmsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File for clicksendsms: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: Basic+ base_url: "https://api.example.com/v1/sms"+ sms_id: G9TXU2YD-NOYB-LLSX-21NU-5CX5SIA330Z7+ sms_email: [email protected]++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "[email protected]")-func TestClickSendsms_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLICKSENDSMS_TOKEN")- inactiveSecret := testSecrets.MustGetField("CLICKSENDSMS_INACTIVE")- email := testSecrets.MustGetField("CLICKSENDSMS_EMAIL")+func TestClickSendSMS_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s", secret, email)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClickSendsms,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clicksendsms secret %s within clicksendsmsemail %s but not valid", inactiveSecret, email)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClickSendsms,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ClickSendsms.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ClickSendsms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.goindex 45d713562..af57edeba 100644--- a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken.go@@ -2,10 +2,11 @@ package clickuppersonaltokenimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_integration_test.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_integration_test.gonew file mode 100644index 000000000..8c8bd60f7--- /dev/null+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package clickuppersonaltoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClickupPersonalToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN")+ inactiveSecret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClickupPersonalToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClickupPersonalToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ClickupPersonalToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ClickupPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.goindex 8c8bd60f7..673c0921a 100644--- a/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go+++ b/pkg/detectors/clickuppersonaltoken/clickuppersonaltoken_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package clickuppersonaltokenimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ base_url: "https://api.example.com/v1/user"+ clickup_token: "pk_7043602_WIKY22PAKCVC1S5Q2X6119IK7N1UL8VY"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "pk_7043602_WIKY22PAKCVC1S5Q2X6119IK7N1UL8VY")-func TestClickupPersonalToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN")- inactiveSecret := testSecrets.MustGetField("CLICKUPPERSONALTOKEN_INACTIVE")+func TestClickupPersonalToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClickupPersonalToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clickuppersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClickupPersonalToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ClickupPersonalToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ClickupPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cliengo/cliengo.go b/pkg/detectors/cliengo/cliengo.goindex 54dcfbad8..3615eef20 100644--- a/pkg/detectors/cliengo/cliengo.go+++ b/pkg/detectors/cliengo/cliengo.go@@ -2,10 +2,11 @@ package cliengoimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cliengo/cliengo_integration_test.go b/pkg/detectors/cliengo/cliengo_integration_test.gonew file mode 100644index 000000000..1d31d5cf0--- /dev/null+++ b/pkg/detectors/cliengo/cliengo_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package cliengo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCliengo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLIENGO")+ inactiveSecret := testSecrets.MustGetField("CLIENGO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cliengo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cliengo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cliengo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cliengo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cliengo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cliengo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/cliengo/cliengo_test.go b/pkg/detectors/cliengo/cliengo_test.goindex 1d31d5cf0..e6250b874 100644--- a/pkg/detectors/cliengo/cliengo_test.go+++ b/pkg/detectors/cliengo/cliengo_test.go@@ -1,116 +1,89 @@-//go:build detectors-// +build detectors-package cliengoimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCliengo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLIENGO")- inactiveSecret := testSecrets.MustGetField("CLIENGO_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.cliengo.com/v1/user?key=9e4635bc-28dc-25d3-8546-2b30115d9a7b"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "9e4635bc-28dc-25d3-8546-2b30115d9a7b"+)++func TestCliengo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cliengo secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cliengo,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cliengo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cliengo,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cliengo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cliengo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/clinchpad/clinchpad.go b/pkg/detectors/clinchpad/clinchpad.goindex 09357e0fe..c39d9727f 100644--- a/pkg/detectors/clinchpad/clinchpad.go+++ b/pkg/detectors/clinchpad/clinchpad.go@@ -4,10 +4,11 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clinchpad/clinchpad_integration_test.go b/pkg/detectors/clinchpad/clinchpad_integration_test.gonew file mode 100644index 000000000..68d05a39f--- /dev/null+++ b/pkg/detectors/clinchpad/clinchpad_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package clinchpad++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClinchpad_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLINCHPAD_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CLINCHPAD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clinchpad,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clinchpad,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Clinchpad.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Clinchpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clinchpad/clinchpad_test.go b/pkg/detectors/clinchpad/clinchpad_test.goindex 68d05a39f..651f1d960 100644--- a/pkg/detectors/clinchpad/clinchpad_test.go+++ b/pkg/detectors/clinchpad/clinchpad_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package clinchpadimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Basic"+ base_url: "https://api.example.com/v1/user"+ clinchpad_key: "3v1xo5r03ghc538iwzbzeddwqulnun8h"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "3v1xo5r03ghc538iwzbzeddwqulnun8h")-func TestClinchpad_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLINCHPAD_TOKEN")- inactiveSecret := testSecrets.MustGetField("CLINCHPAD_INACTIVE")+func TestClinchPad_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clinchpad,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clinchpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clinchpad,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Clinchpad.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Clinchpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clockify/clockify.go b/pkg/detectors/clockify/clockify.goindex e9414f4a7..f89768242 100644--- a/pkg/detectors/clockify/clockify.go+++ b/pkg/detectors/clockify/clockify.go@@ -2,10 +2,11 @@ package clockifyimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clockify/clockify_integration_test.go b/pkg/detectors/clockify/clockify_integration_test.gonew file mode 100644index 000000000..618512566--- /dev/null+++ b/pkg/detectors/clockify/clockify_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package clockify++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClockify_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOCKIFY")+ inactiveSecret := testSecrets.MustGetField("CLOCKIFY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clockify secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clockify,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clockify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Clockify,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Clockify.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Clockify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clockify/clockify_test.go b/pkg/detectors/clockify/clockify_test.goindex 618512566..cff7b09b2 100644--- a/pkg/detectors/clockify/clockify_test.go+++ b/pkg/detectors/clockify/clockify_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package clockifyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.example.com/v1/user"+ clockify_key: "kfJkRn7Knahh6pyDOL82NqNq5c4VLUNulVe5CMyJpIK9NXQC"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "kfJkRn7Knahh6pyDOL82NqNq5c4VLUNulVe5CMyJpIK9NXQC")-func TestClockify_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOCKIFY")- inactiveSecret := testSecrets.MustGetField("CLOCKIFY_INACTIVE")+func TestClockify_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clockify secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clockify,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clockify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Clockify,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Clockify.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Clockify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clockworksms/clockworksms.go b/pkg/detectors/clockworksms/clockworksms.goindex 1b5904893..3924adcc7 100644--- a/pkg/detectors/clockworksms/clockworksms.go+++ b/pkg/detectors/clockworksms/clockworksms.go@@ -2,10 +2,11 @@ package clockworksmsimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clockworksms/clockworksms_integration_test.go b/pkg/detectors/clockworksms/clockworksms_integration_test.gonew file mode 100644index 000000000..4250011b2--- /dev/null+++ b/pkg/detectors/clockworksms/clockworksms_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package clockworksms++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClockworksms_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ userKey := testSecrets.MustGetField("CLOCKWORKSMS_USERKEY")+ token := testSecrets.MustGetField("CLOCKWORKSMS_TOKEN")+ inactiveToken := testSecrets.MustGetField("CLOCKWORKSMS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within", userKey, token)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClockworkSMS,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within but not valid", userKey, inactiveToken)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClockworkSMS,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Clockworksms.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Clockworksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clockworksms/clockworksms_test.go b/pkg/detectors/clockworksms/clockworksms_test.goindex 4250011b2..40f0c3981 100644--- a/pkg/detectors/clockworksms/clockworksms_test.go+++ b/pkg/detectors/clockworksms/clockworksms_test.go@@ -1,121 +1,92 @@-//go:build detectors-// +build detectors-package clockworksmsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ base_url: "https://api.textanywhere.com/v1/user"+ clockwork_key: "84473"+ clockwork_token: "YROh7NZbZxHwiSc9pMNIAGYs"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "84473YROh7NZbZxHwiSc9pMNIAGYs")-func TestClockworksms_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- userKey := testSecrets.MustGetField("CLOCKWORKSMS_USERKEY")- token := testSecrets.MustGetField("CLOCKWORKSMS_TOKEN")- inactiveToken := testSecrets.MustGetField("CLOCKWORKSMS_INACTIVE")+func TestClockWorkSMS_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within", userKey, token)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClockworkSMS,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clockworksms secret %s and clockworksms token %s within but not valid", userKey, inactiveToken)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClockworkSMS,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Clockworksms.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Clockworksms.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/closecrm/close.go b/pkg/detectors/closecrm/close.goindex de0cf4967..00db62924 100644--- a/pkg/detectors/closecrm/close.go+++ b/pkg/detectors/closecrm/close.go@@ -4,10 +4,11 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/closecrm/close_integration_test.go b/pkg/detectors/closecrm/close_integration_test.gonew file mode 100644index 000000000..2a4940dc4--- /dev/null+++ b/pkg/detectors/closecrm/close_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package closecrm++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClose_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOSE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CLOSE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a close secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Close,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a close secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Close,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Close.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Close.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/closecrm/close_test.go b/pkg/detectors/closecrm/close_test.goindex 2a4940dc4..e2a2bb59a 100644--- a/pkg/detectors/closecrm/close_test.go+++ b/pkg/detectors/closecrm/close_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package closecrmimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Basic"+ base_url: "https://api.example.com/v1/user"+ close_key: "api_3cyEW8syFEmeND561qJ9Sj8mT6E0VyWqY7h6cjJIBtc2e"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "api_3cyEW8syFEmeND561qJ9Sj8mT6E0VyWqY7h6cjJIBtc2e")-func TestClose_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOSE_TOKEN")- inactiveSecret := testSecrets.MustGetField("CLOSE_INACTIVE")+func TestCloseCRM_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a close secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Close,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a close secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Close,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Close.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Close.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudconvert/cloudconvert_integration_test.go b/pkg/detectors/cloudconvert/cloudconvert_integration_test.gonew file mode 100644index 000000000..9109b4a58--- /dev/null+++ b/pkg/detectors/cloudconvert/cloudconvert_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cloudconvert++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudConvert_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDCONVERT")+ inactiveSecret := testSecrets.MustGetField("CLOUDCONVERT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudConvert,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudConvert,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CloudConvert.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CloudConvert.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudconvert/cloudconvert_test.go b/pkg/detectors/cloudconvert/cloudconvert_test.goindex 9109b4a58..5caac3f9e 100644--- a/pkg/detectors/cloudconvert/cloudconvert_test.go+++ b/pkg/detectors/cloudconvert/cloudconvert_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package cloudconvertimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ cloudconvert_key: "eypn1gEV3BnckI3jcYzUvliSbukvxzO5acvE?ey8VEaR0lmpRa4IXv02fDPnlfdukWtb1/p-nlQlPVGnB52f9KwY4q98aVghXZqoit4AeFxMAHcCytOj61o8lHdcUF9fIcyF2HaFIk/k3Hdt7pS/5rb2eeWcEvc-5XB0T_Oh68AtCG8mOPpwKvzrhhIuEJck3vtFncgbDrSxg5mKkw924rMLP3Tb5tgIRuawZLwBxJL/qVIhAzfDGiIeNTzYOB9zHfHlfw3aZ1i/terePSN5EafVJ1yYw1KRXWL9/kPdAO0yFwSv3mUWx04oIIUURG6QKwO0rk7L0eAxnu4djnSXtqdvH_G50H1SSwwfKUg2Xz25-OZLkhxiaxEMMMY=3x0Yjhs7O1KFkI5gUQKH_VYAU2bJSAqpCKsxaYrdw91wUoya5rflCBVDHjC/BsezIkPFFmEu7sqs3WJg6dZeAiguYx7uZtDx1ILH18f29q9o34bM9SNolZNcG3fN7L2eWjilbmUq/Ty2545WkbHTjlcjLlHPAAjzLebfcFnlMSKH9Tqb/qx3G1z8wfzMa3dn3iRqNHwfmGOmfgK7RjtlZwoVruMjDWEza/o8imZF513yM7FrHTJkTFa1JjVbjU/C85ItZTiJsBUKAt/DbLg6W7lieKgHbgmz3cuwgVR7YDLZJB056TRcU9wrV0SUYDz0gogrpOEnZxdo4fb5UcCllj/AD/dYsfqVSHtTxKWBhun9Iqmx8FjgPtFCFugTxfaaHZ9dUC7TPahdSxixGvnu8EEvAs0Te85eJ9iyeq628Tvboz9J7KMq/uwflJtecSquJiWJT9GsYL5dl3Hr6ZYhxqs1-mrrB5FNzn-NPclPSu9PANtQ1BDuahKy683/t85F8yjug5C5paamNfgiJgOm5Vi/USUmWeVmH_htZoYGJTbOywDkRT1bYp9JIxlWHA29MInhWNrdlxZ_1h-SQ3fM6pzKIoJ0m_T/KXYERPzle0cy_/OnlfIa-yUgBnx_slQ1f9h0AS/PVMv/yZ6W"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "eypn1gEV3BnckI3jcYzUvliSbukvxzO5acvE?ey8VEaR0lmpRa4IXv02fDPnlfdukWtb1/p-nlQlPVGnB52f9KwY4q98aVghXZqoit4AeFxMAHcCytOj61o8lHdcUF9fIcyF2HaFIk/k3Hdt7pS/5rb2eeWcEvc-5XB0T_Oh68AtCG8mOPpwKvzrhhIuEJck3vtFncgbDrSxg5mKkw924rMLP3Tb5tgIRuawZLwBxJL/qVIhAzfDGiIeNTzYOB9zHfHlfw3aZ1i/terePSN5EafVJ1yYw1KRXWL9/kPdAO0yFwSv3mUWx04oIIUURG6QKwO0rk7L0eAxnu4djnSXtqdvH_G50H1SSwwfKUg2Xz25-OZLkhxiaxEMMMY=3x0Yjhs7O1KFkI5gUQKH_VYAU2bJSAqpCKsxaYrdw91wUoya5rflCBVDHjC/BsezIkPFFmEu7sqs3WJg6dZeAiguYx7uZtDx1ILH18f29q9o34bM9SNolZNcG3fN7L2eWjilbmUq/Ty2545WkbHTjlcjLlHPAAjzLebfcFnlMSKH9Tqb/qx3G1z8wfzMa3dn3iRqNHwfmGOmfgK7RjtlZwoVruMjDWEza/o8imZF513yM7FrHTJkTFa1JjVbjU/C85ItZTiJsBUKAt/DbLg6W7lieKgHbgmz3cuwgVR7YDLZJB056TRcU9wrV0SUYDz0gogrpOEnZxdo4fb5UcCllj/AD/dYsfqVSHtTxKWBhun9Iqmx8FjgPtFCFugTxfaaHZ9dUC7TPahdSxixGvnu8EEvAs0Te85eJ9iyeq628Tvboz9J7KMq/uwflJtecSquJiWJT9GsYL5dl3Hr6ZYhxqs1-mrrB5FNzn-NPclPSu9PANtQ1BDuahKy683/t85F8yjug5C5paamNfgiJgOm5Vi/USUmWeVmH_htZoYGJTbOywDkRT1bYp9JIxlWHA29MInhWNrdlxZ_1h-SQ3fM6pzKIoJ0m_T/KXYERPzle0cy_/OnlfIa-yUgBnx_slQ1f9h0AS/PVMv/yZ6W")-func TestCloudConvert_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDCONVERT")- inactiveSecret := testSecrets.MustGetField("CLOUDCONVERT_INACTIVE")+func TestCloudConvert_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudConvert,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudconvert secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudConvert,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CloudConvert.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CloudConvert.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudelements/cloudelements.go b/pkg/detectors/cloudelements/cloudelements.goindex 9c77ab362..19002fcdb 100644--- a/pkg/detectors/cloudelements/cloudelements.go+++ b/pkg/detectors/cloudelements/cloudelements.go@@ -3,16 +3,17 @@ package cloudelementsimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/cloudelements/cloudelements_integration_test.go b/pkg/detectors/cloudelements/cloudelements_integration_test.gonew file mode 100644index 000000000..6fa4314dd--- /dev/null+++ b/pkg/detectors/cloudelements/cloudelements_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package cloudelements++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudElements_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDELEMENTS")+ org := testSecrets.MustGetField("CLOUDELEMENTS_ORG")+ inactiveSecret := testSecrets.MustGetField("CLOUDELEMENTS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s", secret, org)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudElements,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudElements,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CloudElements.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CloudElements.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudelements/cloudelements_test.go b/pkg/detectors/cloudelements/cloudelements_test.goindex 6fa4314dd..9cda22b9e 100644--- a/pkg/detectors/cloudelements/cloudelements_test.go+++ b/pkg/detectors/cloudelements/cloudelements_test.go@@ -1,121 +1,92 @@-//go:build detectors-// +build detectors-package cloudelementsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ base_url: "https://api.example.com/v1/user"+ cloudelements_key: "4NloL5EzH3PLvNzjCMikofUfKXYOsYOeJBopEyDScIL"+ cloudelements_org: "inz9qofvjwnx59hgefq9sy5v64ilqrnu"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "4NloL5EzH3PLvNzjCMikofUfKXYOsYOeJBopEyDScILinz9qofvjwnx59hgefq9sy5v64ilqrnu")-func TestCloudElements_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDELEMENTS")- org := testSecrets.MustGetField("CLOUDELEMENTS_ORG")- inactiveSecret := testSecrets.MustGetField("CLOUDELEMENTS_INACTIVE")+func TestCloudElements_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s", secret, org)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudElements,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudelements secret %s within cloudelements %s but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudElements,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CloudElements.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CloudElements.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken.goindex 8fa867c6f..18a7bb610 100644--- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken.go@@ -3,10 +3,11 @@ package cloudflareapitokenimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.gonew file mode 100644index 000000000..05660e2c5--- /dev/null+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_integration_test.go@@ -0,0 +1,137 @@+//go:build detectors+// +build detectors++package cloudflareapitoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudflareApiToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_API_INACTIVE")+ secret2 := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN2")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudflareApiToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret2)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudflareApiToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within but unverified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudflareApiToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CloudflareApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CloudflareApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.goindex 05660e2c5..22417496c 100644--- a/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go+++ b/pkg/detectors/cloudflareapitoken/cloudflareapitoken_test.go@@ -1,137 +1,91 @@-//go:build detectors-// +build detectors-package cloudflareapitokenimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ cloudflare_token: "kOjD1yceduu2jxL2uuwT9dkOIudU3_54sLCEud6j"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "kOjD1yceduu2jxL2uuwT9dkOIudU3_54sLCEud6j")-func TestCloudflareApiToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN")- inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_API_INACTIVE")- secret2 := testSecrets.MustGetField("CLOUDFLARE_API_TOKEN2")+func TestCloudFlareAPIToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudflareApiToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within", secret2)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudflareApiToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudflareapitoken secret %s within but unverified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudflareApiToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CloudflareApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CloudflareApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey.go b/pkg/detectors/cloudflarecakey/cloudflarecakey.goindex 69499d35e..d74c59d15 100644--- a/pkg/detectors/cloudflarecakey/cloudflarecakey.go+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey.go@@ -5,10 +5,11 @@ import ("net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"- regexp "github.com/wasilibs/go-re2")type Scanner struct{}diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey_integration_test.go b/pkg/detectors/cloudflarecakey/cloudflarecakey_integration_test.gonew file mode 100644index 000000000..0eedf05b9--- /dev/null+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package cloudflarecakey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudflareCaKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY")+ inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudflareCaKey,+ Verified: true,+ },+ },+ wantErr: false,+ },++ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within but unverified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudflareCaKey,+ Verified: false,+ },+ },+ wantErr: false,+ },++ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CloudflareCaKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CloudflareCaKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.goindex 0eedf05b9..3e1a45ff2 100644--- a/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go+++ b/pkg/detectors/cloudflarecakey/cloudflarecakey_test.go@@ -1,122 +1,91 @@-//go:build detectors-// +build detectors-package cloudflarecakeyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.example.com/v1/user"+ cloudflare: "vR2hGdmAyzLxI4kR34v-fgtaIug7VC_xudRuku6UQd5t9RoiY3zGiuVxMGiN6qR6O0cGjbGepFsHHrfm-9QmBahS.tvHRjYvf_c5XOKReWeVXsp3T-_gilqReVlS-U_9vj1tcMAwRjWCcehKiCOPnXEeN.Xg.lmwp0.mAIsgbrdPJsJE4hz7OF5PQ7D"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "vR2hGdmAyzLxI4kR34v-fgtaIug7VC_xudRuku6UQd5t9RoiY3zGiuVxMGiN6qR6O0cGjbGepFsHHrfm-9QmBahS.tvHRjYvf_c5XOKReWeVXsp3T-_gilqReVlS-U_9vj1tcMAwRjWCcehKiCOPnXEeN.Xg.lmwp0.mAIsgbrdPJsJE4hz7OF5PQ7D")-func TestCloudflareCaKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY")- inactiveSecret := testSecrets.MustGetField("CLOUDFLARE_ORIGIN_CA_KEY_INACTIVE")+func TestCloudFlareCAKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudflareCaKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},+ }- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudflarecakey secret %s within but unverified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudflareCaKey,- Verified: false,- },- },- wantErr: false,- },+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CloudflareCaKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CloudflareCaKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudimage/cloudimage.go b/pkg/detectors/cloudimage/cloudimage.goindex 51a02c0f1..06bd2905e 100644--- a/pkg/detectors/cloudimage/cloudimage.go+++ b/pkg/detectors/cloudimage/cloudimage.go@@ -2,11 +2,12 @@ package cloudimageimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings""time"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cloudimage/cloudimage_integration_test.go b/pkg/detectors/cloudimage/cloudimage_integration_test.gonew file mode 100644index 000000000..00e00f173--- /dev/null+++ b/pkg/detectors/cloudimage/cloudimage_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package cloudimage++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudImage_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDIMAGE")+ inactiveSecret := testSecrets.MustGetField("CLOUDIMAGE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudImage,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CloudImage,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CloudImage.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CloudImage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/cloudimage/cloudimage_test.go b/pkg/detectors/cloudimage/cloudimage_test.goindex 00e00f173..0c329a4ec 100644--- a/pkg/detectors/cloudimage/cloudimage_test.go+++ b/pkg/detectors/cloudimage/cloudimage_test.go@@ -1,116 +1,90 @@-//go:build detectors-// +build detectors-package cloudimageimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCloudImage_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDIMAGE")- inactiveSecret := testSecrets.MustGetField("CLOUDIMAGE_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ base_url: "https://api.example.com/v1/user"+ cloudimage: "d__9rvli8sm4jo18v5q0q4n7vhkwbv"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "d__9rvli8sm4jo18v5q0q4n7vhkwbv"+)++func TestCloudImage_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudImage,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudimage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CloudImage,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CloudImage.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CloudImage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/cloudmersive/cloudmersive.go b/pkg/detectors/cloudmersive/cloudmersive.goindex b4ef56fc1..36cc9ae58 100644--- a/pkg/detectors/cloudmersive/cloudmersive.go+++ b/pkg/detectors/cloudmersive/cloudmersive.go@@ -2,10 +2,11 @@ package cloudmersiveimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cloudmersive/cloudmersive_integration_test.go b/pkg/detectors/cloudmersive/cloudmersive_integration_test.gonew file mode 100644index 000000000..e0acea511--- /dev/null+++ b/pkg/detectors/cloudmersive/cloudmersive_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cloudmersive++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudmersive_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDMERSIVE")+ inactiveSecret := testSecrets.MustGetField("CLOUDMERSIVE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloudmersive,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloudmersive,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cloudmersive.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cloudmersive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudmersive/cloudmersive_test.go b/pkg/detectors/cloudmersive/cloudmersive_test.goindex e0acea511..c8187c1fb 100644--- a/pkg/detectors/cloudmersive/cloudmersive_test.go+++ b/pkg/detectors/cloudmersive/cloudmersive_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package cloudmersiveimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.example.com/v1/user"+ cloudmersive: "sxk5k1nfra8jak0mjjc6afr6v-6gsf7dr9o1"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "sxk5k1nfra8jak0mjjc6afr6v-6gsf7dr9o1")-func TestCloudmersive_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDMERSIVE")- inactiveSecret := testSecrets.MustGetField("CLOUDMERSIVE_INACTIVE")+func TestCloudMersive_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloudmersive,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudmersive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloudmersive,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cloudmersive.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cloudmersive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudplan/cloudplan_integration_test.go b/pkg/detectors/cloudplan/cloudplan_integration_test.gonew file mode 100644index 000000000..531d959b4--- /dev/null+++ b/pkg/detectors/cloudplan/cloudplan_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cloudplan++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudplan_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDPLAN")+ inactiveSecret := testSecrets.MustGetField("CLOUDPLAN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloudplan,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloudplan,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cloudplan.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cloudplan.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudplan/cloudplan_test.go b/pkg/detectors/cloudplan/cloudplan_test.goindex 531d959b4..ab2f5c9b5 100644--- a/pkg/detectors/cloudplan/cloudplan_test.go+++ b/pkg/detectors/cloudplan/cloudplan_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package cloudplanimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ base_url: "https://api.example.com/v1/user"+ cloudplan_session_key: "Y6D1FIS3XZXIJLKD82P6U8IXYV4UEYPP"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "Y6D1FIS3XZXIJLKD82P6U8IXYV4UEYPP")-func TestCloudplan_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDPLAN")- inactiveSecret := testSecrets.MustGetField("CLOUDPLAN_INACTIVE")+func TestCloudPlan_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloudplan,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudplan secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloudplan,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cloudplan.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cloudplan.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloudsmith/cloudsmith_integration_test.go b/pkg/detectors/cloudsmith/cloudsmith_integration_test.gonew file mode 100644index 000000000..58fe57574--- /dev/null+++ b/pkg/detectors/cloudsmith/cloudsmith_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cloudsmith++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloudsmith_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOUDSMITH")+ inactiveSecret := testSecrets.MustGetField("CLOUDSMITH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloudsmith,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloudsmith,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cloudsmith.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cloudsmith.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloudsmith/cloudsmith_test.go b/pkg/detectors/cloudsmith/cloudsmith_test.goindex 58fe57574..86aa492dd 100644--- a/pkg/detectors/cloudsmith/cloudsmith_test.go+++ b/pkg/detectors/cloudsmith/cloudsmith_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package cloudsmithimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "X-API-Key"+ base_url: "https://api.example.com/v1/user"+ cloudsmith: "6fd00a2cfd7bbc51e1c4db6ac2f29d59629afd22"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "6fd00a2cfd7bbc51e1c4db6ac2f29d59629afd22")-func TestCloudsmith_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOUDSMITH")- inactiveSecret := testSecrets.MustGetField("CLOUDSMITH_INACTIVE")+func TestCloudSmith_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloudsmith,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloudsmith secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloudsmith,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cloudsmith.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cloudsmith.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cloverly/cloverly.go b/pkg/detectors/cloverly/cloverly.goindex 6cadfbb5e..3d17b1323 100644--- a/pkg/detectors/cloverly/cloverly.go+++ b/pkg/detectors/cloverly/cloverly.go@@ -3,10 +3,11 @@ package cloverlyimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cloverly/cloverly_integration_test.go b/pkg/detectors/cloverly/cloverly_integration_test.gonew file mode 100644index 000000000..38d6b72a0--- /dev/null+++ b/pkg/detectors/cloverly/cloverly_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cloverly++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCloverly_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLOVERLY")+ inactiveSecret := testSecrets.MustGetField("CLOVERLY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloverly secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloverly,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cloverly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Cloverly,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Cloverly.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Cloverly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cloverly/cloverly_test.go b/pkg/detectors/cloverly/cloverly_test.goindex cb4d64794..c1929ed64 100644--- a/pkg/detectors/cloverly/cloverly_test.go+++ b/pkg/detectors/cloverly/cloverly_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package cloverlyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ cloverly_token: "564i_0a9_v58bn:p9st3r3cgi95_"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "564i_0a9_v58bn:p9st3r3cgi95_")-func TestCloverly_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLOVERLY")- inactiveSecret := testSecrets.MustGetField("CLOVERLY_INACTIVE")+func TestCloverly_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloverly secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloverly,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cloverly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Cloverly,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Cloverly.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Cloverly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/clustdoc/clustdoc.go b/pkg/detectors/clustdoc/clustdoc.goindex cd1f32866..14d70d695 100644--- a/pkg/detectors/clustdoc/clustdoc.go+++ b/pkg/detectors/clustdoc/clustdoc.go@@ -3,10 +3,11 @@ package clustdocimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/clustdoc/clustdoc_integration_test.go b/pkg/detectors/clustdoc/clustdoc_integration_test.gonew file mode 100644index 000000000..35802efce--- /dev/null+++ b/pkg/detectors/clustdoc/clustdoc_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package clustdoc++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestClustDoc_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CLUSTDOC")+ inactiveSecret := testSecrets.MustGetField("CLUSTDOC_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClustDoc,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ClustDoc,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ClustDoc.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ClustDoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/clustdoc/clustdoc_test.go b/pkg/detectors/clustdoc/clustdoc_test.goindex 35802efce..d3f08c59a 100644--- a/pkg/detectors/clustdoc/clustdoc_test.go+++ b/pkg/detectors/clustdoc/clustdoc_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package clustdocimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ clustdoc_token: "yQ7mTTO4eJ4I9GHDEdzF3wq0KVowNKjPMud3q0ZqaEIuoR1qCrARUyLwknNP"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "yQ7mTTO4eJ4I9GHDEdzF3wq0KVowNKjPMud3q0ZqaEIuoR1qCrARUyLwknNP")-func TestClustDoc_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CLUSTDOC")- inactiveSecret := testSecrets.MustGetField("CLUSTDOC_INACTIVE")+func TestClustDoc_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClustDoc,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a clustdoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ClustDoc,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ClustDoc.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ClustDoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/coda/coda.go b/pkg/detectors/coda/coda.goindex 74e0b6ce7..c4c32d72d 100644--- a/pkg/detectors/coda/coda.go+++ b/pkg/detectors/coda/coda.go@@ -3,10 +3,11 @@ package codaimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/coda/coda_integration_test.go b/pkg/detectors/coda/coda_integration_test.gonew file mode 100644index 000000000..4b38faab8--- /dev/null+++ b/pkg/detectors/coda/coda_integration_test.go@@ -0,0 +1,161 @@+//go:build detectors+// +build detectors++package coda++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCoda_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CODA")+ inactiveSecret := testSecrets.MustGetField("CODA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coda,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coda secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coda,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coda,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coda,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Coda.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Coda.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/coda/coda_test.go b/pkg/detectors/coda/coda_test.goindex e260351d3..217b783ad 100644--- a/pkg/detectors/coda/coda_test.go+++ b/pkg/detectors/coda/coda_test.go@@ -1,161 +1,90 @@-//go:build detectors-// +build detectors-package codaimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ coda_token: "64ukni4l-zub4-3coe-html-9byb40oi5i87"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "64ukni4l-zub4-3coe-html-9byb40oi5i87")-func TestCoda_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CODA")- inactiveSecret := testSecrets.MustGetField("CODA_INACTIVE")+func TestCoda_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coda,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coda secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coda,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coda,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coda secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coda,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Coda.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Coda.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/codacy/codacy.go b/pkg/detectors/codacy/codacy.goindex 20b086a52..bbe419713 100644--- a/pkg/detectors/codacy/codacy.go+++ b/pkg/detectors/codacy/codacy.go@@ -2,10 +2,11 @@ package codacyimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/codacy/codacy_integration_test.go b/pkg/detectors/codacy/codacy_integration_test.gonew file mode 100644index 000000000..d086024ef--- /dev/null+++ b/pkg/detectors/codacy/codacy_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package codacy++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCodacy_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CODACY")+ inactiveSecret := testSecrets.MustGetField("CODACY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codacy secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codacy,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codacy,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Codacy.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Codacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/codacy/codacy_test.go b/pkg/detectors/codacy/codacy_test.goindex d086024ef..f416a30e3 100644--- a/pkg/detectors/codacy/codacy_test.go+++ b/pkg/detectors/codacy/codacy_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package codacyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ codacy_token: "g73RSmTIzTU1wUA5BXYI"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "g73RSmTIzTU1wUA5BXYI")-func TestCodacy_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CODACY")- inactiveSecret := testSecrets.MustGetField("CODACY_INACTIVE")+func TestCodacy_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codacy secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codacy,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codacy,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Codacy.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Codacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/codeclimate/codeclimate.go b/pkg/detectors/codeclimate/codeclimate.goindex 8916cc2de..9b7e4055a 100644--- a/pkg/detectors/codeclimate/codeclimate.go+++ b/pkg/detectors/codeclimate/codeclimate.go@@ -4,10 +4,11 @@ import ("context""encoding/json""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/codeclimate/codeclimate_integration_test.go b/pkg/detectors/codeclimate/codeclimate_integration_test.gonew file mode 100644index 000000000..3a2fcf1b1--- /dev/null+++ b/pkg/detectors/codeclimate/codeclimate_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package codeclimate++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCodeclimate_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CODECLIMATE")+ inactiveSecret := testSecrets.MustGetField("CODECLIMATE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codeclimate,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codeclimate,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Codeclimate.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Codeclimate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/codeclimate/codeclimate_test.go b/pkg/detectors/codeclimate/codeclimate_test.goindex 2bbffcb25..2a4c6b0ac 100644--- a/pkg/detectors/codeclimate/codeclimate_test.go+++ b/pkg/detectors/codeclimate/codeclimate_test.go@@ -2,116 +2,90 @@ package codeclimateimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ api:+ auth_type: "API-Key"+ base_url: "https://api.example.com/v1/user"+ codeclimate_token: "efbc069555c703d31c3bcc6fbd426cec5f21eb43"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "efbc069555c703d31c3bcc6fbd426cec5f21eb43")-func TestCodeclimate_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CODECLIMATE")- inactiveSecret := testSecrets.MustGetField("CODECLIMATE_INACTIVE")+func TestCodeClimate_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codeclimate,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codeclimate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codeclimate,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Codeclimate.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Codeclimate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/codemagic/codemagic.go b/pkg/detectors/codemagic/codemagic.goindex 499137292..561f7ba13 100644--- a/pkg/detectors/codemagic/codemagic.go+++ b/pkg/detectors/codemagic/codemagic.go@@ -2,10 +2,11 @@ package codemagicimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/codemagic/codemagic_integration_test.go b/pkg/detectors/codemagic/codemagic_integration_test.gonew file mode 100644index 000000000..e3ab148ae--- /dev/null+++ b/pkg/detectors/codemagic/codemagic_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package codemagic++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCodemagic_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CODEMAGIC")+ inactiveSecret := testSecrets.MustGetField("CODEMAGIC_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codemagic secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codemagic,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codemagic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codemagic,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Codemagic.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Codemagic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/codemagic/codemagic_test.go b/pkg/detectors/codemagic/codemagic_test.goindex e3ab148ae..e110566af 100644--- a/pkg/detectors/codemagic/codemagic_test.go+++ b/pkg/detectors/codemagic/codemagic_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package codemagicimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Auth-Key"+ base_url: "https://api.example.com/v1/user"+ codemagic_key: "PSIYbVgfkbEPQoqJfzHpACTtihONkQ_cKmOpNDPNiCU"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "PSIYbVgfkbEPQoqJfzHpACTtihONkQ_cKmOpNDPNiCU")-func TestCodemagic_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CODEMAGIC")- inactiveSecret := testSecrets.MustGetField("CODEMAGIC_INACTIVE")+func TestCodeMagic_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codemagic secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codemagic,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codemagic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codemagic,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Codemagic.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Codemagic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/codequiry/codequiry.go b/pkg/detectors/codequiry/codequiry.goindex 0c5b64d36..94dbe6e27 100644--- a/pkg/detectors/codequiry/codequiry.go+++ b/pkg/detectors/codequiry/codequiry.go@@ -2,10 +2,11 @@ package codequiryimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/codequiry/codequiry_integration_test.go b/pkg/detectors/codequiry/codequiry_integration_test.gonew file mode 100644index 000000000..30e9e1751--- /dev/null+++ b/pkg/detectors/codequiry/codequiry_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package codequiry++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCodequiry_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CODEQUIRY")+ inactiveSecret := testSecrets.MustGetField("CODEQUIRY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codequiry secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codequiry,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a codequiry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Codequiry,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Codequiry.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Codequiry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/codequiry/codequiry_test.go b/pkg/detectors/codequiry/codequiry_test.goindex 30e9e1751..268555c38 100644--- a/pkg/detectors/codequiry/codequiry_test.go+++ b/pkg/detectors/codequiry/codequiry_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package codequiryimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.example.com/v1/user"+ codequiry_key: "7cA6eb3AvmlVSqLMP4iKvp7fXtEAADQud11KidjPzbSLcvntAD8CK6P7uGaGdlit"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "7cA6eb3AvmlVSqLMP4iKvp7fXtEAADQud11KidjPzbSLcvntAD8CK6P7uGaGdlit")-func TestCodequiry_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CODEQUIRY")- inactiveSecret := testSecrets.MustGetField("CODEQUIRY_INACTIVE")+func TestCodeQuiry_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codequiry secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codequiry,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a codequiry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Codequiry,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Codequiry.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Codequiry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/coinapi/coinapi.go b/pkg/detectors/coinapi/coinapi.goindex df7886f79..ad53f8916 100644--- a/pkg/detectors/coinapi/coinapi.go+++ b/pkg/detectors/coinapi/coinapi.go@@ -2,10 +2,11 @@ package coinapiimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/coinapi/coinapi_integration_test.go b/pkg/detectors/coinapi/coinapi_integration_test.gonew file mode 100644index 000000000..9ee35f7f9--- /dev/null+++ b/pkg/detectors/coinapi/coinapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package coinapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCoinApi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COINAPI")+ inactiveSecret := testSecrets.MustGetField("COINAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CoinApi,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CoinApi,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CoinApi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CoinApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/coinapi/coinapi_test.go b/pkg/detectors/coinapi/coinapi_test.goindex 9ee35f7f9..a13dd6ce0 100644--- a/pkg/detectors/coinapi/coinapi_test.go+++ b/pkg/detectors/coinapi/coinapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package coinapiimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.example.com/v1/user"+ coinapi_key: "6D8B5AUIRDQCB3NRKSMZZL9RCV9G07GTHUR3"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "6D8B5AUIRDQCB3NRKSMZZL9RCV9G07GTHUR3")-func TestCoinApi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COINAPI")- inactiveSecret := testSecrets.MustGetField("COINAPI_INACTIVE")+func TestCoinAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coinapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CoinApi,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coinapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CoinApi,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CoinApi.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CoinApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/coinbase/coinbase.go b/pkg/detectors/coinbase/coinbase.goindex c1eabc672..81db4a4df 100644--- a/pkg/detectors/coinbase/coinbase.go+++ b/pkg/detectors/coinbase/coinbase.go@@ -3,10 +3,11 @@ package coinbaseimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/coinbase/coinbase_integration_test.go b/pkg/detectors/coinbase/coinbase_integration_test.gonew file mode 100644index 000000000..12a2cfa39--- /dev/null+++ b/pkg/detectors/coinbase/coinbase_integration_test.go@@ -0,0 +1,116 @@+//go:build detectors+// +build detectors++package coinbase++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCoinbase_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COINBASE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("COINBASE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinbase secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coinbase,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinbase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coinbase,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Coinbase.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Coinbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/coinbase/coinbase_test.go b/pkg/detectors/coinbase/coinbase_test.goindex 5bb99e187..430c41899 100644--- a/pkg/detectors/coinbase/coinbase_test.go+++ b/pkg/detectors/coinbase/coinbase_test.go@@ -1,116 +1,91 @@-//go:build detectors-// +build detectors-package coinbase-// import (-// "context"-// "fmt"-// "testing"-// "time"+import (+ "context"+ "testing"++ "github.com/google/go-cmp/cmp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ base_url: "https://api.example.com/v1/user"+ coinbase_key: "IIQle6bXgoQEzxqHBt2VN0gW-Yve3t5k5VVD3nNAUUMCLVdK-3M4k6apjs43l0nM"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "IIQle6bXgoQEzxqHBt2VN0gW-Yve3t5k5VVD3nNAUUMCLVdK-3M4k6apjs43l0nM"+)++func TestCoinBase_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})++ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},+ },+ }-// "github.com/kylelemons/godebug/pretty"+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }-// "github.com/trufflesecurity/trufflehog/v3/pkg/common"-// "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-// "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-// )+ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }-// func TestCoinbase_FromChunk(t *testing.T) {-// ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)-// defer cancel()-// testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")-// if err != nil {-// t.Fatalf("could not get test secrets from GCP: %s", err)-// }-// secret := testSecrets.MustGetField("COINBASE_TOKEN")-// inactiveSecret := testSecrets.MustGetField("COINBASE_INACTIVE")+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }-// type args struct {-// ctx context.Context-// data []byte-// verify bool-// }-// tests := []struct {-// name string-// s Scanner-// args args-// want []detectors.Result-// wantErr bool-// }{-// {-// name: "found, verified",-// s: Scanner{},-// args: args{-// ctx: context.Background(),-// data: []byte(fmt.Sprintf("You can find a coinbase secret %s within", secret)),-// verify: true,-// },-// want: []detectors.Result{-// {-// DetectorType: detectorspb.DetectorType_Coinbase,-// Verified: true,-// },-// },-// wantErr: false,-// },-// {-// name: "found, unverified",-// s: Scanner{},-// args: args{-// ctx: context.Background(),-// data: []byte(fmt.Sprintf(""You can find a coinbase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation-// verify: true,-// },-// want: []detectors.Result{-// {-// DetectorType: detectorspb.DetectorType_Coinbase,-// Verified: false,-// },-// },-// wantErr: false,-// },-// {-// name: "not found",-// s: Scanner{},-// args: args{-// ctx: context.Background(),-// data: []byte("You cannot find the secret within"),-// verify: true,-// },-// want: nil,-// wantErr: false,-// },-// }-// for _, tt := range tests {-// t.Run(tt.name, func(t *testing.T) {-// s := Scanner{}-// got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)-// if (err != nil) != tt.wantErr {-// t.Errorf("Coinbase.FromData() error = %v, wantErr %v", err, tt.wantErr)-// return-// }-// for i := range got {-// if len(got[i].Raw) == 0 {-// t.Fatal("no raw secret present")-// }-// got[i].Raw = nil-// }-// if diff := pretty.Compare(got, tt.want); diff != "" {-// t.Errorf("Coinbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)-// }-// })-// }-// }+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }-// func BenchmarkFromData(benchmark *testing.B) {-// ctx := context.Background()-// s := Scanner{}-// for name, data := range detectors.MustGetBenchmarkData() {-// benchmark.Run(name, func(b *testing.B) {-// for n := 0; n < b.N; n++ {-// s.FromData(ctx, false, data)-// }-// })-// }-// }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}diff --git a/pkg/detectors/coinlayer/coinlayer.go b/pkg/detectors/coinlayer/coinlayer.goindex ade604ad3..ba043d25c 100644--- a/pkg/detectors/coinlayer/coinlayer.go+++ b/pkg/detectors/coinlayer/coinlayer.go@@ -3,11 +3,12 @@ package coinlayerimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/coinlayer/coinlayer_integration_test.go b/pkg/detectors/coinlayer/coinlayer_integration_test.gonew file mode 100644index 000000000..095501355--- /dev/null+++ b/pkg/detectors/coinlayer/coinlayer_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package coinlayer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCoinlayer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COINLAYER")+ inactiveSecret := testSecrets.MustGetField("COINLAYER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coinlayer,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coinlayer,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Coinlayer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Coinlayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/coinlayer/coinlayer_test.go b/pkg/detectors/coinlayer/coinlayer_test.goindex 095501355..15960cce9 100644--- a/pkg/detectors/coinlayer/coinlayer_test.go+++ b/pkg/detectors/coinlayer/coinlayer_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package coinlayerimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.coinlayer.com/v1/user?key=gg2einqoe3zxu0ju7c3wqg9vql662vdj"+ key: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "gg2einqoe3zxu0ju7c3wqg9vql662vdj")-func TestCoinlayer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COINLAYER")- inactiveSecret := testSecrets.MustGetField("COINLAYER_INACTIVE")+func TestCoinLayer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coinlayer,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coinlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coinlayer,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Coinlayer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Coinlayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/coinlib/coinlib.go b/pkg/detectors/coinlib/coinlib.goindex aca9a734c..92053ed40 100644--- a/pkg/detectors/coinlib/coinlib.go+++ b/pkg/detectors/coinlib/coinlib.go@@ -3,10 +3,11 @@ package coinlibimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/coinlib/coinlib_integration_test.go b/pkg/detectors/coinlib/coinlib_integration_test.gonew file mode 100644index 000000000..a49554487--- /dev/null+++ b/pkg/detectors/coinlib/coinlib_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package coinlib++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCoinlib_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COINLIB")+ inactiveSecret := testSecrets.MustGetField("COINLIB_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinlib secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coinlib,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coinlib secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coinlib,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Coinlib.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Coinlib.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/coinlib/coinlib_test.go b/pkg/detectors/coinlib/coinlib_test.goindex bdcc00c2b..eba2c008a 100644--- a/pkg/detectors/coinlib/coinlib_test.go+++ b/pkg/detectors/coinlib/coinlib_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package coinlibimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.coinlib.com/v1/user?key=seugeupfknprstoe"+ key: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "seugeupfknprstoe")-func TestCoinlib_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COINLIB")- inactiveSecret := testSecrets.MustGetField("COINLIB_INACTIVE")+func TestCoinLib_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coinlib secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coinlib,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coinlib secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coinlib,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Coinlib.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Coinlib.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/collect2/collect2_integration_test.go b/pkg/detectors/collect2/collect2_integration_test.gonew file mode 100644index 000000000..ddd91500e--- /dev/null+++ b/pkg/detectors/collect2/collect2_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package collect2++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCollect2_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COLLECT2")+ inactiveSecret := testSecrets.MustGetField("COLLECT2_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a collect2 secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Collect2,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a collect2 secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Collect2,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Collect2.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Collect2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/collect2/collect2_test.go b/pkg/detectors/collect2/collect2_test.goindex 64f058a09..c65c7e1dc 100644--- a/pkg/detectors/collect2/collect2_test.go+++ b/pkg/detectors/collect2/collect2_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package collect2import ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ base_url: "https://api.collect2.com/v1/user?key=22f39f53-3bd4-d84b-8e29-00402d5c316f"+ key: ""++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "22f39f53-3bd4-d84b-8e29-00402d5c316f")-func TestCollect2_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COLLECT2")- inactiveSecret := testSecrets.MustGetField("COLLECT2_INACTIVE")+func TestCollect2_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a collect2 secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Collect2,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a collect2 secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Collect2,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Collect2.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Collect2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/column/column_integration_test.go b/pkg/detectors/column/column_integration_test.gonew file mode 100644index 000000000..6e9b22e4f--- /dev/null+++ b/pkg/detectors/column/column_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package column++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestColumn_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COLUMN_ACTIVE")+ inactiveSecret := testSecrets.MustGetField("COLUMN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a column secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Column,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a column secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Column,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Column.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Column.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/column/column_test.go b/pkg/detectors/column/column_test.goindex 6e9b22e4f..8f19087d3 100644--- a/pkg/detectors/column/column_test.go+++ b/pkg/detectors/column/column_test.go@@ -1,116 +1,90 @@-//go:build detectors-// +build detectors-package columnimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestColumn_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COLUMN_ACTIVE")- inactiveSecret := testSecrets.MustGetField("COLUMN_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Basic"+ base_url: "https://api.example.com/v1/user"+ column_key: "live_ID8Jxlu0QRsV7rKkWI9CUDpkrUv"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "live_ID8Jxlu0QRsV7rKkWI9CUDpkrUv"+)++func TestColumn_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a column secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Column,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a column secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Column,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Column.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Column.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/commercejs/commercejs.go b/pkg/detectors/commercejs/commercejs.goindex d7af6292a..781fd4591 100644--- a/pkg/detectors/commercejs/commercejs.go+++ b/pkg/detectors/commercejs/commercejs.go@@ -2,10 +2,11 @@ package commercejsimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/commercejs/commercejs_integration_test.go b/pkg/detectors/commercejs/commercejs_integration_test.gonew file mode 100644index 000000000..0c39e067e--- /dev/null+++ b/pkg/detectors/commercejs/commercejs_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package commercejs++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCommerceJS_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COMMERCEJS")+ inactiveSecret := testSecrets.MustGetField("COMMERCEJS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a commercejs secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CommerceJS,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a commercejs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CommerceJS,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CommerceJS.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CommerceJS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/commercejs/commercejs_test.go b/pkg/detectors/commercejs/commercejs_test.goindex 0c39e067e..57363764a 100644--- a/pkg/detectors/commercejs/commercejs_test.go+++ b/pkg/detectors/commercejs/commercejs_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package commercejsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ in: "Header"+ base_url: "https://api.example.com/v1/user"+ commercejs_key: "g6cl4jt_2noyibalgbqid4h58jxivqdgyyxovepbvqmbl7wq"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "g6cl4jt_2noyibalgbqid4h58jxivqdgyyxovepbvqmbl7wq")-func TestCommerceJS_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COMMERCEJS")- inactiveSecret := testSecrets.MustGetField("COMMERCEJS_INACTIVE")+func TestCommerceJS_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a commercejs secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CommerceJS,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a commercejs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CommerceJS,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CommerceJS.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CommerceJS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/commodities/commodities_integration_test.go b/pkg/detectors/commodities/commodities_integration_test.gonew file mode 100644index 000000000..0c0ef7c2d--- /dev/null+++ b/pkg/detectors/commodities/commodities_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package commodities++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCommodities_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COMMODITIES")+ inactiveSecret := testSecrets.MustGetField("COMMODITIES_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a commodities secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Commodities,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a commodities secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Commodities,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Commodities.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Commodities.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/commodities/commodities_test.go b/pkg/detectors/commodities/commodities_test.goindex 0c0ef7c2d..cb857f603 100644--- a/pkg/detectors/commodities/commodities_test.go+++ b/pkg/detectors/commodities/commodities_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package commoditiesimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ commodities_key: "5aJKBqkCyGCT9FIWNUTmowbzqgcm9DUCi60mHwgPQRBSt7dFahv9eY329Dn9"+ base_url: "https://api.example.com/v1/user?access_key=$commodities_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "5aJKBqkCyGCT9FIWNUTmowbzqgcm9DUCi60mHwgPQRBSt7dFahv9eY329Dn9")-func TestCommodities_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COMMODITIES")- inactiveSecret := testSecrets.MustGetField("COMMODITIES_INACTIVE")+func TestCommodities_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a commodities secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Commodities,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a commodities secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Commodities,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Commodities.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Commodities.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/companyhub/companyhub.go b/pkg/detectors/companyhub/companyhub.goindex ead1f778e..6350835b3 100644--- a/pkg/detectors/companyhub/companyhub.go+++ b/pkg/detectors/companyhub/companyhub.go@@ -3,10 +3,11 @@ package companyhubimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/companyhub/companyhub_integration_test.go b/pkg/detectors/companyhub/companyhub_integration_test.gonew file mode 100644index 000000000..877eef42e--- /dev/null+++ b/pkg/detectors/companyhub/companyhub_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package companyhub++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCompanyHub_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COMPANYHUB_TOKEN")+ inactiveSecret := testSecrets.MustGetField("COMPANYHUB_INACTIVE")+ user := testSecrets.MustGetField("COMPANYHUB_USER")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s", secret, user)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CompanyHub,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CompanyHub,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CompanyHub.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CompanyHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/companyhub/companyhub_test.go b/pkg/detectors/companyhub/companyhub_test.goindex 877eef42e..eb2c7faf8 100644--- a/pkg/detectors/companyhub/companyhub_test.go+++ b/pkg/detectors/companyhub/companyhub_test.go@@ -1,121 +1,96 @@-//go:build detectors-// +build detectors-package companyhubimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCompanyHub_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COMPANYHUB_TOKEN")- inactiveSecret := testSecrets.MustGetField("COMPANYHUB_INACTIVE")- user := testSecrets.MustGetField("COMPANYHUB_USER")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly- type args struct {- ctx context.Context- data []byte- verify bool+ api:+ auth_type: ""+ in: "Header"+ companyhub_key: "A5zrYt9xY4X1Q9mG6IX6"+ companyhub_id: "xzAMMncOTR7^5d5NS6"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secrets = []string{+ "A5zrYt9xY4X1Q9mG6IX6xzAMMncOTR7^5d5NS6",+ "A5zrYt9xY4X1Q9mG6IX6A5zrYt9xY4X1Q9mG6IX6",}+)++func TestCompanyHub_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s", secret, user)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CompanyHub,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a companyhub secret %s within companyhubuser %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CompanyHub,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: secrets,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CompanyHub.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CompanyHub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/confluent/confluent.go b/pkg/detectors/confluent/confluent.goindex 82d34e52f..ea73d2bd8 100644--- a/pkg/detectors/confluent/confluent.go+++ b/pkg/detectors/confluent/confluent.go@@ -4,16 +4,17 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/confluent/confluent_integration_test.go b/pkg/detectors/confluent/confluent_integration_test.gonew file mode 100644index 000000000..91b535dec--- /dev/null+++ b/pkg/detectors/confluent/confluent_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package confluent++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestConfluent_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CONFLUENT_TOKEN")+ key := testSecrets.MustGetField("CONFLUENT_KEY")+ inactiveSecret := testSecrets.MustGetField("CONFLUENT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s", secret, key)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Confluent,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Confluent,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Confluent.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Confluent.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/confluent/confluent_test.go b/pkg/detectors/confluent/confluent_test.goindex 91b535dec..2d541a112 100644--- a/pkg/detectors/confluent/confluent_test.go+++ b/pkg/detectors/confluent/confluent_test.go@@ -1,121 +1,93 @@-//go:build detectors-// +build detectors-package confluentimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Basic"+ in: "Header"+ confluent_key: "CVAJHB4RAZboV3Od"+ confluent_secret: "pIsdFuG0oJuyiir3GWqpC4pv7xpKFodCNh6PYN4XdE8EtyIwYtzEer0KtHQ8kofs"+ base_url: "https://api.example.com/v1/user?api-key=$confluent_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "CVAJHB4RAZboV3OdpIsdFuG0oJuyiir3GWqpC4pv7xpKFodCNh6PYN4XdE8EtyIwYtzEer0KtHQ8kofs")-func TestConfluent_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CONFLUENT_TOKEN")- key := testSecrets.MustGetField("CONFLUENT_KEY")- inactiveSecret := testSecrets.MustGetField("CONFLUENT_INACTIVE")+func TestConfluent_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s", secret, key)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Confluent,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a confluent secret %s within confluent %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Confluent,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Confluent.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Confluent.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.goindex ecd70e57a..e8004a608 100644--- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken.go@@ -3,10 +3,11 @@ package contentfulpersonalaccesstokenimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.goindex f69286595..52c2b55b5 100644--- a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalaccesstoken_test.go@@ -1,121 +1,91 @@-//go:build detectors-// +build detectors-package contentfulpersonalaccesstokenimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ in: "Header"+ contentful_access_token: "CFPAT-"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `)-func TestContentfulPersonalAccessToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN")- inactiveSecret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN_INACTIVE")+func TestContentfulPersonalAccessToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a contentful secret %s within contentful https://api.contentful.com/organizations", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a contentful secret %s within but unverified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - not found",+ input: validPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ContentfulPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ContentfulPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalacesstoken_integration_test.go b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalacesstoken_integration_test.gonew file mode 100644index 000000000..f69286595--- /dev/null+++ b/pkg/detectors/contentfulpersonalaccesstoken/contentfulpersonalacesstoken_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package contentfulpersonalaccesstoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestContentfulPersonalAccessToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN")+ inactiveSecret := testSecrets.MustGetField("CONTENTFULPERSONALACCESSTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a contentful secret %s within contentful https://api.contentful.com/organizations", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a contentful secret %s within but unverified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ContentfulPersonalAccessToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ContentfulPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {++ t.Errorf("ContentfulPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/conversiontools/conversiontools.go b/pkg/detectors/conversiontools/conversiontools.goindex f1b012007..f4732bf38 100644--- a/pkg/detectors/conversiontools/conversiontools.go+++ b/pkg/detectors/conversiontools/conversiontools.go@@ -3,10 +3,11 @@ package conversiontoolsimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/conversiontools/conversiontools_integration_test.go b/pkg/detectors/conversiontools/conversiontools_integration_test.gonew file mode 100644index 000000000..ab698d0a2--- /dev/null+++ b/pkg/detectors/conversiontools/conversiontools_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package conversiontools++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestConversionTools_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CONVERSIONTOOLS")+ inactiveSecret := testSecrets.MustGetField("CONVERSIONTOOLS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ConversionTools,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ConversionTools,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ConversionTools.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ConversionTools.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/conversiontools/conversiontools_test.go b/pkg/detectors/conversiontools/conversiontools_test.goindex 748c3567c..ff0893a10 100644--- a/pkg/detectors/conversiontools/conversiontools_test.go+++ b/pkg/detectors/conversiontools/conversiontools_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package conversiontoolsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ in: "Header"+ conversiontools_key: "ey5g5C73ichf2TWwQQfuPNG2SW1xdTmCHFgS6zsUjRz3kkiEofoa8X7SVGjwAMkhrv5KyOFqunP29gQpKq9A4sPF_Ps4B4IkTtgUG9cgP5A5ygAkuSR2rsOC.SIDSLIy4jZiL7L8ZHyAhyR8msV7JzxlI6YsNqmmj"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "ey5g5C73ichf2TWwQQfuPNG2SW1xdTmCHFgS6zsUjRz3kkiEofoa8X7SVGjwAMkhrv5KyOFqunP29gQpKq9A4sPF_Ps4B4IkTtgUG9cgP5A5ygAkuSR2rsOC.SIDSLIy4jZiL7L8ZHyAhyR8msV7JzxlI6YsNqmmj")-func TestConversionTools_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CONVERSIONTOOLS")- inactiveSecret := testSecrets.MustGetField("CONVERSIONTOOLS_INACTIVE")+func TestConversionTools_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ConversionTools,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a conversiontools secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ConversionTools,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ConversionTools.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ConversionTools.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/convertapi/convertapi.go b/pkg/detectors/convertapi/convertapi.goindex 600ea2dbe..06dbe7d1b 100644--- a/pkg/detectors/convertapi/convertapi.go+++ b/pkg/detectors/convertapi/convertapi.go@@ -3,10 +3,11 @@ package convertapiimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/convertapi/convertapi_integration_test.go b/pkg/detectors/convertapi/convertapi_integration_test.gonew file mode 100644index 000000000..89acb1db5--- /dev/null+++ b/pkg/detectors/convertapi/convertapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package convertapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestConvertApi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CONVERTAPI")+ inactiveSecret := testSecrets.MustGetField("CONVERTAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a convertapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ConvertApi,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a convertapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ConvertApi,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ConvertApi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ConvertApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/convertapi/convertapi_test.go b/pkg/detectors/convertapi/convertapi_test.goindex 89acb1db5..869a552d6 100644--- a/pkg/detectors/convertapi/convertapi_test.go+++ b/pkg/detectors/convertapi/convertapi_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package convertapiimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ convertapi_key: "H9ZGTfAERfN5W0AX"+ base_url: "https://api.example.com/v1/user?Secret=$convertapi_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "H9ZGTfAERfN5W0AX")-func TestConvertApi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CONVERTAPI")- inactiveSecret := testSecrets.MustGetField("CONVERTAPI_INACTIVE")+func TestConvertAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a convertapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ConvertApi,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a convertapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ConvertApi,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ConvertApi.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ConvertApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/convertkit/convertkit.go b/pkg/detectors/convertkit/convertkit.goindex 1f915d8a2..35cb1f139 100644--- a/pkg/detectors/convertkit/convertkit.go+++ b/pkg/detectors/convertkit/convertkit.go@@ -2,10 +2,11 @@ package convertkitimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/convertkit/convertkit_integration_test.go b/pkg/detectors/convertkit/convertkit_integration_test.gonew file mode 100644index 000000000..2b773d28c--- /dev/null+++ b/pkg/detectors/convertkit/convertkit_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package convertkit++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestConvertkit_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CONVERTKIT_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CONVERTKIT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a convertkit secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Convertkit,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a convertkit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Convertkit,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Convertkit.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Convertkit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/convertkit/convertkit_test.go b/pkg/detectors/convertkit/convertkit_test.goindex 2b773d28c..5909f9f5b 100644--- a/pkg/detectors/convertkit/convertkit_test.go+++ b/pkg/detectors/convertkit/convertkit_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package convertkitimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ convertkit_key: "hfCnuVcYOgiRjlDEmAoRbN"+ base_url: "https://api.example.com/v1/forms?api-key=$convertapi_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "hfCnuVcYOgiRjlDEmAoRbN")-func TestConvertkit_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CONVERTKIT_TOKEN")- inactiveSecret := testSecrets.MustGetField("CONVERTKIT_INACTIVE")+func TestConvertKit_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a convertkit secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Convertkit,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a convertkit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Convertkit,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Convertkit.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Convertkit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/convier/convier_integration_test.go b/pkg/detectors/convier/convier_integration_test.gonew file mode 100644index 000000000..190274a00--- /dev/null+++ b/pkg/detectors/convier/convier_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package convier++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestConvier_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CONVIER")+ inactiveSecret := testSecrets.MustGetField("CONVIER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a convier secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Convier,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a convier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Convier,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Convier.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Convier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/convier/convier_test.go b/pkg/detectors/convier/convier_test.goindex 190274a00..86d1d76b3 100644--- a/pkg/detectors/convier/convier_test.go+++ b/pkg/detectors/convier/convier_test.go@@ -1,116 +1,91 @@-//go:build detectors-// +build detectors-package convierimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestConvier_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CONVIER")- inactiveSecret := testSecrets.MustGetField("CONVIER_INACTIVE")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ in: "Header"+ convier_key: "49|07KJBwfPzF2ESyNui5yBw9OVB6eWj0iXkssEKC7b"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "49|07KJBwfPzF2ESyNui5yBw9OVB6eWj0iXkssEKC7b"+)++func TestConvier_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a convier secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Convier,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a convier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Convier,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Convier.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Convier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/copper/copper.go b/pkg/detectors/copper/copper.goindex 60ca33216..ff1ec3c95 100644--- a/pkg/detectors/copper/copper.go+++ b/pkg/detectors/copper/copper.go@@ -2,10 +2,11 @@ package copperimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/copper/copper_integration_test.go b/pkg/detectors/copper/copper_integration_test.gonew file mode 100644index 000000000..9849e144a--- /dev/null+++ b/pkg/detectors/copper/copper_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package copper++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCopper_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COOPER_TOKEN")+ inactiveSecret := testSecrets.MustGetField("COOPER_INACTIVE")+ id := testSecrets.MustGetField("COOPER_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Copper,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Copper,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Copper.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Copper.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/copper/copper_test.go b/pkg/detectors/copper/copper_test.goindex cf7d9f021..e6f9944ef 100644--- a/pkg/detectors/copper/copper_test.go+++ b/pkg/detectors/copper/copper_test.go@@ -1,121 +1,93 @@-//go:build detectors-// +build detectors-package copperimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ in: "Header"+ copper_email: "s0ovh@P8I~p3"+ copper_token: "noqs39jzqaegbam2k6mai9ov1uwsl21y"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "noqs39jzqaegbam2k6mai9ov1uwsl21ys0ovh@P8I~p3")-func TestCopper_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COOPER_TOKEN")- inactiveSecret := testSecrets.MustGetField("COOPER_INACTIVE")- id := testSecrets.MustGetField("COOPER_ID")+func TestCopper_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Copper,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a copper secret %s within copperid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Copper,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Copper.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Copper.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/couchbase/couchbase.go b/pkg/detectors/couchbase/couchbase.goindex 43e60e3de..2d47de59c 100644--- a/pkg/detectors/couchbase/couchbase.go+++ b/pkg/detectors/couchbase/couchbase.go@@ -15,7 +15,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/couchbase/couchbase_integration_test.go b/pkg/detectors/couchbase/couchbase_integration_test.gonew file mode 100644index 000000000..03b340879--- /dev/null+++ b/pkg/detectors/couchbase/couchbase_integration_test.go@@ -0,0 +1,123 @@+//go:build detectors+// +build detectors++package couchbase++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCouchbase_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")+ username := testSecrets.MustGetField("COUCHBASE_USERNAME")+ password := testSecrets.MustGetField("COUCHBASE_PASSWORD")+ inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Couchbase,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Couchbase,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}++ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/couchbase/couchbase_test.go b/pkg/detectors/couchbase/couchbase_test.goindex 03b340879..9412bdbad 100644--- a/pkg/detectors/couchbase/couchbase_test.go+++ b/pkg/detectors/couchbase/couchbase_test.go@@ -1,123 +1,97 @@-//go:build detectors-// +build detectors-package couchbaseimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCouchbase_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")- username := testSecrets.MustGetField("COUCHBASE_USERNAME")- password := testSecrets.MustGetField("COUCHBASE_PASSWORD")- inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Password"+ in: "Configuration"+ couchbase_domain: "cb.testing.cloud.couchbase.com" // couchbase://+ couchbase_username: "usrpS@d>p"+ couchbase_password: "passwordU+2028 rf\@V[4,L/?2}"+ base_url: "https://$couchbase_domain/v1/user"- type args struct {- ctx context.Context- data []byte- verify bool+ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secrets = []string{+ `usrpS@d>p:passwordU+2028@couchbases://cb.testing.cloud.couchbase.com`,+ `$DB_USERNAME:passwordU+2028@couchbases://cb.testing.cloud.couchbase.com`,}+)++func TestCouchBase_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Couchbase,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Couchbase,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: secrets,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/countrylayer/countrylayer.go b/pkg/detectors/countrylayer/countrylayer.goindex cee4ca102..57a402597 100644--- a/pkg/detectors/countrylayer/countrylayer.go+++ b/pkg/detectors/countrylayer/countrylayer.go@@ -3,10 +3,11 @@ package countrylayerimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/countrylayer/countrylayer_integration_test.go b/pkg/detectors/countrylayer/countrylayer_integration_test.gonew file mode 100644index 000000000..c70dcc65a--- /dev/null+++ b/pkg/detectors/countrylayer/countrylayer_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package countrylayer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCountryLayer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COUNTRYLAYER")+ inactiveSecret := testSecrets.MustGetField("COUNTRYLAYER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CountryLayer,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CountryLayer,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CountryLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CountryLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/countrylayer/countrylayer_test.go b/pkg/detectors/countrylayer/countrylayer_test.goindex c70dcc65a..f97fd88cd 100644--- a/pkg/detectors/countrylayer/countrylayer_test.go+++ b/pkg/detectors/countrylayer/countrylayer_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package countrylayerimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ countrylayer_key: "031eiaqplnq39py5ppsctxo6n2xj5t10"+ base_url: "https://api.example.com/v1/user?access_key=$countrylayer_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "031eiaqplnq39py5ppsctxo6n2xj5t10")-func TestCountryLayer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COUNTRYLAYER")- inactiveSecret := testSecrets.MustGetField("COUNTRYLAYER_INACTIVE")+func TestCountryLayer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CountryLayer,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a countrylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CountryLayer,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CountryLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CountryLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/courier/courier.go b/pkg/detectors/courier/courier.goindex 822fe2e2c..96db0c1c1 100644--- a/pkg/detectors/courier/courier.go+++ b/pkg/detectors/courier/courier.go@@ -3,10 +3,11 @@ package courierimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/courier/courier_integration_test.go b/pkg/detectors/courier/courier_integration_test.gonew file mode 100644index 000000000..2b6f81803--- /dev/null+++ b/pkg/detectors/courier/courier_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package courier++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCourier_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COURIER")+ inactiveSecret := testSecrets.MustGetField("COURIER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a courier secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Courier,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a courier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Courier,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Courier.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Courier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/courier/courier_test.go b/pkg/detectors/courier/courier_test.goindex 2b6f81803..b7c90a976 100644--- a/pkg/detectors/courier/courier_test.go+++ b/pkg/detectors/courier/courier_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package courierimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ in: "Header"+ courier_key: "pk_iHWk6NqTne0QthfSVF7uixZpa3OTYpA8hC6bIIavhluXfxz37FB_rHPqJNWh06HpNIOuokET4dfFzXS1"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "pk_iHWk6NqTne0QthfSVF7uixZpa3OTYpA8hC6bIIavhluXfxz37FB_rHPqJNWh06HpNIOuokET4dfFzXS1")-func TestCourier_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COURIER")- inactiveSecret := testSecrets.MustGetField("COURIER_INACTIVE")+func TestCourier_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a courier secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Courier,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a courier secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Courier,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Courier.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Courier.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/coveralls/coveralls.go b/pkg/detectors/coveralls/coveralls.goindex 17e412fb0..f22fb29e3 100644--- a/pkg/detectors/coveralls/coveralls.go+++ b/pkg/detectors/coveralls/coveralls.go@@ -3,10 +3,11 @@ package coverallsimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/coveralls/coveralls_integration_test.go b/pkg/detectors/coveralls/coveralls_integration_test.gonew file mode 100644index 000000000..17b7299de--- /dev/null+++ b/pkg/detectors/coveralls/coveralls_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package coveralls++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCoveralls_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("COVERALLS_TOKEN")+ inactiveSecret := testSecrets.MustGetField("COVERALLS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coveralls secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coveralls,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a coveralls secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Coveralls,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Coveralls.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Coveralls.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/coveralls/coveralls_test.go b/pkg/detectors/coveralls/coveralls_test.goindex 2b20f8ece..32e36173e 100644--- a/pkg/detectors/coveralls/coveralls_test.go+++ b/pkg/detectors/coveralls/coveralls_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package coverallsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ in: "Header"+ coveralls_token: "tPfhjkzKJyWtUdxDLYMjNDEfP7Yn9WvWb2-K3"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "tPfhjkzKJyWtUdxDLYMjNDEfP7Yn9WvWb2-K3")-func TestCoveralls_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("COVERALLS_TOKEN")- inactiveSecret := testSecrets.MustGetField("COVERALLS_INACTIVE")+func TestCoveralls_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coveralls secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coveralls,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a coveralls secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Coveralls,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Coveralls.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Coveralls.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/craftmypdf/craftmypdf_integration_test.go b/pkg/detectors/craftmypdf/craftmypdf_integration_test.gonew file mode 100644index 000000000..b2952d214--- /dev/null+++ b/pkg/detectors/craftmypdf/craftmypdf_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package craftmypdf++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCraftMyPDF_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CRAFTMYPDF")+ inactiveSecret := testSecrets.MustGetField("CRAFTMYPDF_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CraftMyPDF,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CraftMyPDF,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CraftMyPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CraftMyPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/craftmypdf/craftmypdf_test.go b/pkg/detectors/craftmypdf/craftmypdf_test.goindex b2952d214..2df3db9f5 100644--- a/pkg/detectors/craftmypdf/craftmypdf_test.go+++ b/pkg/detectors/craftmypdf/craftmypdf_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package craftmypdfimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Header"+ craftmypdf_key: "GuTSS3XQdT6fx00mxudKq7oj2CsieZCGmEc"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "GuTSS3XQdT6fx00mxudKq7oj2CsieZCGmEc")-func TestCraftMyPDF_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CRAFTMYPDF")- inactiveSecret := testSecrets.MustGetField("CRAFTMYPDF_INACTIVE")+func TestCraftMyPDF_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CraftMyPDF,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a craftmypdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CraftMyPDF,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CraftMyPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CraftMyPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/crowdin/crowdin.go b/pkg/detectors/crowdin/crowdin.goindex 57875afc1..ff3073cf6 100644--- a/pkg/detectors/crowdin/crowdin.go+++ b/pkg/detectors/crowdin/crowdin.go@@ -3,10 +3,11 @@ package crowdinimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/crowdin/crowdin_integration_test.go b/pkg/detectors/crowdin/crowdin_integration_test.gonew file mode 100644index 000000000..10a050d8c--- /dev/null+++ b/pkg/detectors/crowdin/crowdin_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package crowdin++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCrowdin_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CROWDIN")+ inactiveSecret := testSecrets.MustGetField("CROWDIN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a crowdin secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Crowdin,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a crowdin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Crowdin,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Crowdin.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Crowdin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/crowdin/crowdin_test.go b/pkg/detectors/crowdin/crowdin_test.goindex 10a050d8c..fab9962c4 100644--- a/pkg/detectors/crowdin/crowdin_test.go+++ b/pkg/detectors/crowdin/crowdin_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package crowdinimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "Bearer"+ in: "Header"+ crowdin_token: "BiIRgdPvboWwqlhQtlnCsM041zVYCJ5yMfgltWesDiu9bv1nuRtCEPewsDL3vgRFcp2qLemaPMa8L9g7"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "BiIRgdPvboWwqlhQtlnCsM041zVYCJ5yMfgltWesDiu9bv1nuRtCEPewsDL3vgRFcp2qLemaPMa8L9g7")-func TestCrowdin_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CROWDIN")- inactiveSecret := testSecrets.MustGetField("CROWDIN_INACTIVE")+func TestCrowDin_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a crowdin secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Crowdin,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a crowdin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Crowdin,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Crowdin.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Crowdin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/cryptocompare/cryptocompare.go b/pkg/detectors/cryptocompare/cryptocompare.goindex a1cfd7f6c..02974f301 100644--- a/pkg/detectors/cryptocompare/cryptocompare.go+++ b/pkg/detectors/cryptocompare/cryptocompare.go@@ -2,11 +2,12 @@ package cryptocompareimport ("context"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/cryptocompare/cryptocompare_integration_test.go b/pkg/detectors/cryptocompare/cryptocompare_integration_test.gonew file mode 100644index 000000000..49f069a3c--- /dev/null+++ b/pkg/detectors/cryptocompare/cryptocompare_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package cryptocompare++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCryptoCompare_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CRYPTOCOMPARE")+ inactiveSecret := testSecrets.MustGetField("CRYPTOCOMPARE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CryptoCompare,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CryptoCompare,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CryptoCompare.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CryptoCompare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/cryptocompare/cryptocompare_test.go b/pkg/detectors/cryptocompare/cryptocompare_test.goindex 49f069a3c..9f0a319d0 100644--- a/pkg/detectors/cryptocompare/cryptocompare_test.go+++ b/pkg/detectors/cryptocompare/cryptocompare_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package cryptocompareimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Header"+ cryptocompare_key: "lx8zzovs5h15zl15mj224zks2v25re59965gz0l1z4jsc0bng33a75m5pf52-bvd"+ base_url: "https://api.example.com/v1/user?api_key=$cryptocompare_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "lx8zzovs5h15zl15mj224zks2v25re59965gz0l1z4jsc0bng33a75m5pf52-bvd")-func TestCryptoCompare_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CRYPTOCOMPARE")- inactiveSecret := testSecrets.MustGetField("CRYPTOCOMPARE_INACTIVE")+func TestCryptoCompare_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CryptoCompare,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a cryptocompare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CryptoCompare,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CryptoCompare.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CryptoCompare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/currencyfreaks/currencyfreaks.go b/pkg/detectors/currencyfreaks/currencyfreaks.goindex b4306d84b..5efc9c7c1 100644--- a/pkg/detectors/currencyfreaks/currencyfreaks.go+++ b/pkg/detectors/currencyfreaks/currencyfreaks.go@@ -2,10 +2,11 @@ package currencyfreaksimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/currencyfreaks/currencyfreaks_integration_test.go b/pkg/detectors/currencyfreaks/currencyfreaks_integration_test.gonew file mode 100644index 000000000..e3daa56b1--- /dev/null+++ b/pkg/detectors/currencyfreaks/currencyfreaks_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package currencyfreaks++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCurrencyfreaks_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CURRENCYFREAKS")+ inactiveSecret := testSecrets.MustGetField("CURRENCYFREAKS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Currencyfreaks,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Currencyfreaks,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Currencyfreaks.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Currencyfreaks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/currencyfreaks/currencyfreaks_test.go b/pkg/detectors/currencyfreaks/currencyfreaks_test.goindex e3daa56b1..8349dd400 100644--- a/pkg/detectors/currencyfreaks/currencyfreaks_test.go+++ b/pkg/detectors/currencyfreaks/currencyfreaks_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package currencyfreaksimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Header"+ currencyfreaks_key: "6zlrpo4u8z4s72b2nqr54m9ehmqvwe8p"+ base_url: "https://api.example.com/v1/user?apiKey=$currencyfreaks_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "6zlrpo4u8z4s72b2nqr54m9ehmqvwe8p")-func TestCurrencyfreaks_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CURRENCYFREAKS")- inactiveSecret := testSecrets.MustGetField("CURRENCYFREAKS_INACTIVE")+func TestCurrencyFreaks_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Currencyfreaks,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currencyfreaks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Currencyfreaks,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Currencyfreaks.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Currencyfreaks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/currencylayer/currencylayer.go b/pkg/detectors/currencylayer/currencylayer.goindex fdede2bfe..3bb7966ce 100644--- a/pkg/detectors/currencylayer/currencylayer.go+++ b/pkg/detectors/currencylayer/currencylayer.go@@ -3,11 +3,12 @@ package currencylayerimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/currencylayer/currencylayer_integration_test.go b/pkg/detectors/currencylayer/currencylayer_integration_test.gonew file mode 100644index 000000000..5cc7958d8--- /dev/null+++ b/pkg/detectors/currencylayer/currencylayer_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package currencylayer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCurrencylayer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CURRENCYLAYER")+ inactiveSecret := testSecrets.MustGetField("CURRENCYLAYER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Currencylayer,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Currencylayer,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Currencylayer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Currencylayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/currencylayer/currencylayer_test.go b/pkg/detectors/currencylayer/currencylayer_test.goindex 5cc7958d8..bc8d98ea0 100644--- a/pkg/detectors/currencylayer/currencylayer_test.go+++ b/pkg/detectors/currencylayer/currencylayer_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package currencylayerimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ currencylayer_key: "sxthwp257vpusfe4gr4d4awc794lkxvh"+ base_url: "https://api.example.com/v1/user?access_key=$currencylayer_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "sxthwp257vpusfe4gr4d4awc794lkxvh")-func TestCurrencylayer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CURRENCYLAYER")- inactiveSecret := testSecrets.MustGetField("CURRENCYLAYER_INACTIVE")+func TestCurrencyLayer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Currencylayer,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currencylayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Currencylayer,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Currencylayer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Currencylayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/currencyscoop/currencyscoop.go b/pkg/detectors/currencyscoop/currencyscoop.goindex 539371b4a..452d29900 100644--- a/pkg/detectors/currencyscoop/currencyscoop.go+++ b/pkg/detectors/currencyscoop/currencyscoop.go@@ -3,10 +3,11 @@ package currencyscoopimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/currencyscoop/currencyscoop_integration_test.go b/pkg/detectors/currencyscoop/currencyscoop_integration_test.gonew file mode 100644index 000000000..378a11cff--- /dev/null+++ b/pkg/detectors/currencyscoop/currencyscoop_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package currencyscoop++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCurrencyscoop_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CURRENCYSCOOP")+ inactiveSecret := testSecrets.MustGetField("CURRENCYSCOOP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CurrencyScoop,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CurrencyScoop,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Currencyscoop.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Currencyscoop.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/currencyscoop/currencyscoop_test.go b/pkg/detectors/currencyscoop/currencyscoop_test.goindex 378a11cff..08eebc55b 100644--- a/pkg/detectors/currencyscoop/currencyscoop_test.go+++ b/pkg/detectors/currencyscoop/currencyscoop_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package currencyscoopimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ currencyscoop_key: "70x6tezndca5dqlm5tnn7s03bm6c27jt"+ base_url: "https://api.example.com/v1/user?api_key=$currencylayer_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "70x6tezndca5dqlm5tnn7s03bm6c27jt")-func TestCurrencyscoop_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CURRENCYSCOOP")- inactiveSecret := testSecrets.MustGetField("CURRENCYSCOOP_INACTIVE")+func TestCurrencyScoop_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CurrencyScoop,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currencyscoop secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CurrencyScoop,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Currencyscoop.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Currencyscoop.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/currentsapi/currentsapi.go b/pkg/detectors/currentsapi/currentsapi.goindex 782eb7c22..ae9d3a7fe 100644--- a/pkg/detectors/currentsapi/currentsapi.go+++ b/pkg/detectors/currentsapi/currentsapi.go@@ -2,10 +2,11 @@ package currentsapiimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/currentsapi/currentsapi_integration_test.go b/pkg/detectors/currentsapi/currentsapi_integration_test.gonew file mode 100644index 000000000..63a8e8ed5--- /dev/null+++ b/pkg/detectors/currentsapi/currentsapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package currentsapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCurrentsAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CURRENTSAPI")+ inactiveSecret := testSecrets.MustGetField("CURRENTSAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CurrentsAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CurrentsAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CurrentsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CurrentsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/currentsapi/currentsapi_test.go b/pkg/detectors/currentsapi/currentsapi_test.goindex 63a8e8ed5..63351932b 100644--- a/pkg/detectors/currentsapi/currentsapi_test.go+++ b/pkg/detectors/currentsapi/currentsapi_test.go@@ -1,120 +1,92 @@-//go:build detectors-// +build detectors-package currentsapiimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: ""+ in: "Header"+ currentsapi_key: "thisistheapikey(asaspijf89aa$)lkunjo0#aenoiecnjh"+ base_url: "https://api.example.com/v1/user"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "thisistheapikey(asaspijf89aa$)lkunjo0#aenoiecnjh")-func TestCurrentsAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CURRENTSAPI")- inactiveSecret := testSecrets.MustGetField("CURRENTSAPI_INACTIVE")+func TestCurrentsAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CurrentsAPI,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a currentsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CurrentsAPI,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CurrentsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CurrentsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/customerguru/customerguru.go b/pkg/detectors/customerguru/customerguru.goindex 1a6d87ad3..a7fe1690c 100644--- a/pkg/detectors/customerguru/customerguru.go+++ b/pkg/detectors/customerguru/customerguru.go@@ -2,16 +2,17 @@ package customerguruimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}diff --git a/pkg/detectors/customerguru/customerguru_integration_test.go b/pkg/detectors/customerguru/customerguru_integration_test.gonew file mode 100644index 000000000..3c16c76aa--- /dev/null+++ b/pkg/detectors/customerguru/customerguru_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package customerguru++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCustomerGuru_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CUSTOMERGURU_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CUSTOMERGURU_INACTIVE")+ key := testSecrets.MustGetField("CUSTOMERGURU_KEY")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s", secret, key)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CustomerGuru,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CustomerGuru,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CustomerGuru.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CustomerGuru.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/customerguru/customerguru_test.go b/pkg/detectors/customerguru/customerguru_test.goindex 3c16c76aa..960a8557f 100644--- a/pkg/detectors/customerguru/customerguru_test.go+++ b/pkg/detectors/customerguru/customerguru_test.go@@ -1,121 +1,93 @@-//go:build detectors-// +build detectors-package customerguruimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly++ api:+ auth_type: "API-Key"+ in: "Path"+ guru_key: "WWj2zAK0tMkVJqc28Itfu6THQycyfT"+ guru_id: "Ic53IHpPK71wIacbCgEkIlFbw0VIMcsz6ir2i2DJ0XDRdirf2K"+ base_url: "https://api.customerguru.com/v1/user?api_secret=$guru_id&api_token=guru_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secret = "WWj2zAK0tMkVJqc28Itfu6THQycyfTIc53IHpPK71wIacbCgEkIlFbw0VIMcsz6ir2i2DJ0XDRdirf2K")-func TestCustomerGuru_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CUSTOMERGURU_TOKEN")- inactiveSecret := testSecrets.MustGetField("CUSTOMERGURU_INACTIVE")- key := testSecrets.MustGetField("CUSTOMERGURU_KEY")+func TestCustomerGuru_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s", secret, key)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CustomerGuru,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a customerguru secret %s within customergurukey %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CustomerGuru,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{secret},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CustomerGuru.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CustomerGuru.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/customerio/customerio.go b/pkg/detectors/customerio/customerio.goindex 349d4c01c..770228a23 100644--- a/pkg/detectors/customerio/customerio.go+++ b/pkg/detectors/customerio/customerio.go@@ -4,10 +4,11 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/customerio/customerio_integration_test.go b/pkg/detectors/customerio/customerio_integration_test.gonew file mode 100644index 000000000..a28613e04--- /dev/null+++ b/pkg/detectors/customerio/customerio_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package customerio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestCustomerIO_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("CUSTOMERIO_TOKEN")+ inactiveSecret := testSecrets.MustGetField("CUSTOMERIO_INACTIVE")+ id := testSecrets.MustGetField("CUSTOMERIO_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s ", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CustomerIO,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_CustomerIO,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("CustomerIO.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("CustomerIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/customerio/customerio_test.go b/pkg/detectors/customerio/customerio_test.goindex a28613e04..e8cbf150b 100644--- a/pkg/detectors/customerio/customerio_test.go+++ b/pkg/detectors/customerio/customerio_test.go@@ -1,121 +1,98 @@-//go:build detectors-// +build detectors-package customerioimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestCustomerIO_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("CUSTOMERIO_TOKEN")- inactiveSecret := testSecrets.MustGetField("CUSTOMERIO_INACTIVE")- id := testSecrets.MustGetField("CUSTOMERIO_ID")+var (+ validPattern = `+ # Configuration File: config.yaml+ database:+ host: $DB_HOST+ port: $DB_PORT+ username: $DB_USERNAME+ password: $DB_PASS # IMPORTANT: Do not share this password publicly- type args struct {- ctx context.Context- data []byte- verify bool+ api:+ auth_type: "Basic"+ in: "Header"+ customerio_key: "bXQLU0kcl0A7kxCErc3L"+ customerio_id: "tM2JFc8pmKHUmkdwhmgG"+ base_url: "https://api.example.com/v1/user?access_key=$currencylayer_key"++ # Notes:+ # - Remember to rotate the secret every 90 days.+ # - The above credentials should only be used in a secure environment.+ `+ secrets = []string{+ "bXQLU0kcl0A7kxCErc3LbXQLU0kcl0A7kxCErc3L",+ "bXQLU0kcl0A7kxCErc3LtM2JFc8pmKHUmkdwhmgG",+ "tM2JFc8pmKHUmkdwhmgGbXQLU0kcl0A7kxCErc3L",+ "tM2JFc8pmKHUmkdwhmgGtM2JFc8pmKHUmkdwhmgG",}+)++func TestCustomerio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s ", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CustomerIO,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a customerio secret %s within customerid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_CustomerIO,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: secrets,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("CustomerIO.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("CustomerIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}