Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Added Pattern test cases for detectors (#3354)
Kashif Khana4cc5f7cc38c
diff --git a/pkg/detectors/abbysale/abbysale_integration_test.go b/pkg/detectors/abbysale/abbysale_integration_test.gonew file mode 100644index 000000000..665ce0b81--- /dev/null+++ b/pkg/detectors/abbysale/abbysale_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package abbysale++import (+ "context"+ "fmt"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+)++func TestAbbysale_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ABBYSALE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("ABBYSALE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Abbysale,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Abbysale,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Abbysale,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Abbysale,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }++ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/abbysale/abbysale_test.go b/pkg/detectors/abbysale/abbysale_test.goindex 8f4744985..329a3eca0 100644--- a/pkg/detectors/abbysale/abbysale_test.go+++ b/pkg/detectors/abbysale/abbysale_test.go@@ -1,167 +1,85 @@-//go:build detectors-// +build detectors-package abbysaleimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "123abcDEF456ghiJKL789mnoPQR012stuVWX3456"+ invalidPattern = "123abcDEF456ghiJKL789mnoPQR012stuVWX")-func TestAbbysale_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ABBYSALE_TOKEN")- inactiveSecret := testSecrets.MustGetField("ABBYSALE_INACTIVE")+func TestAbbySale_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Abbysale,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Abbysale,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Abbysale,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("abbysale token = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Abbysale,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("abbysale token keyword is not close to the real token = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("abbysale = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}+ return+ }- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/abstract/abstract_integration_test.go b/pkg/detectors/abstract/abstract_integration_test.gonew file mode 100644index 000000000..2ed3939c9--- /dev/null+++ b/pkg/detectors/abstract/abstract_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package abstract++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAbstract_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ABSTRACT")+ inactiveSecret := testSecrets.MustGetField("ABSTRACT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Abstract,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Abstract,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Abstract,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Abstract,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ time.Sleep(900 * time.Millisecond) // avoid rate limiting+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Abstract.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Abstract.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/abstract/abstract_test.go b/pkg/detectors/abstract/abstract_test.goindex 2ed3939c9..647c1e564 100644--- a/pkg/detectors/abstract/abstract_test.go+++ b/pkg/detectors/abstract/abstract_test.go@@ -1,167 +1,85 @@-//go:build detectors-// +build detectors-package abstractimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAbstract_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ABSTRACT")- inactiveSecret := testSecrets.MustGetField("ABSTRACT_INACTIVE")+var (+ validPattern = "qwerty12345iugt67s7a7sa0akhsxz82"+ invalidPattern = "zxcvbr12345iugt67s7a7sa0akhsXz820"+)++func TestAbstract_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Abstract,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("abstract token = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Abstract,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("abstract token keyword is not close to the real token = '%s'", validPattern),+ want: nil,},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Abstract,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Abstract,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("abstract = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- time.Sleep(900 * time.Millisecond) // avoid rate limiting- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Abstract.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Abstract.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/abuseipdb/abuseipdb_integration_test.go b/pkg/detectors/abuseipdb/abuseipdb_integration_test.gonew file mode 100644index 000000000..bd481f080--- /dev/null+++ b/pkg/detectors/abuseipdb/abuseipdb_integration_test.go@@ -0,0 +1,167 @@+//go:build detectors+// +build detectors++package abuseipdb++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAbuseIPDB_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ABUSEIPDB")+ inactiveSecret := testSecrets.MustGetField("ABUSEIPDB_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but verified", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AbuseIPDB,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_AbuseIPDB,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_AbuseIPDB,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AbuseIPDB,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AbuseIPDB.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AbuseIPDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/abuseipdb/abuseipdb_test.go b/pkg/detectors/abuseipdb/abuseipdb_test.goindex bd481f080..eec2bf0fd 100644--- a/pkg/detectors/abuseipdb/abuseipdb_test.go+++ b/pkg/detectors/abuseipdb/abuseipdb_test.go@@ -1,166 +1,85 @@-//go:build detectors-// +build detectors-package abuseipdbimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAbuseIPDB_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ABUSEIPDB")- inactiveSecret := testSecrets.MustGetField("ABUSEIPDB_INACTIVE")+var (+ validPattern = "123abcdef456ghijkl789mnopqr012stuvwx3455123abcdef456ghijkl789mnopqr012stuvwx3455"+ invalidPattern = "123abcdef456Ghijkl789mnopqr012stuvwx3455123abcdef456ghijkl789mnopqr012stuvwX"+)++func TestAbuseipdb_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but verified", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AbuseIPDB,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("abuseipdb token = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_AbuseIPDB,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("abuseipdb token keyword is not close to the real token = '%s'", validPattern),+ want: nil,},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_AbuseIPDB,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AbuseIPDB,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("abuseipdb = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AbuseIPDB.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AbuseIPDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/accuweather/accuweather_integration_test.go b/pkg/detectors/accuweather/accuweather_integration_test.gonew file mode 100644index 000000000..8dc6f8421--- /dev/null+++ b/pkg/detectors/accuweather/accuweather_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package accuweather++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAccuweather_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ACCUWEATHER")+ inactiveSecret := testSecrets.MustGetField("ACCUWEATHER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Accuweather,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Accuweather,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Accuweather,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Accuweather,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Accuweather.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ got[i].Raw = nil+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Accuweather.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/accuweather/accuweather_test.go b/pkg/detectors/accuweather/accuweather_test.goindex 8dc6f8421..a862f0b0e 100644--- a/pkg/detectors/accuweather/accuweather_test.go+++ b/pkg/detectors/accuweather/accuweather_test.go@@ -1,167 +1,85 @@-//go:build detectors-// +build detectors-package accuweatherimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAccuweather_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ACCUWEATHER")- inactiveSecret := testSecrets.MustGetField("ACCUWEATHER_INACTIVE")+var (+ validPattern = "dqftwc490oPc%xae67sBSF741M56%sd091a"+ invalidPattern = "dqftwc490oPc%xae67sBSF741M56=sd091a"+)++func TestAccuWeather_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Accuweather,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("accuweather token = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Accuweather,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("accuweather token keyword is not close to the real token = '%s'", validPattern),+ want: nil,},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Accuweather,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Accuweather,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("accuweather = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Accuweather.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}- got[i].Raw = nil}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Accuweather.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/adobeio/adobeio_integration_test.go b/pkg/detectors/adobeio/adobeio_integration_test.gonew file mode 100644index 000000000..e1e3914b1--- /dev/null+++ b/pkg/detectors/adobeio/adobeio_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package adobeio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAdobeIO_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ADOBEIO_TOKEN")+ id := testSecrets.MustGetField("ADOBEIO_PRODUCT")+ inactiveSecret := testSecrets.MustGetField("ADOBEIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AdobeIO,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AdobeIO,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AdobeIO.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AdobeIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/adobeio/adobeio_test.go b/pkg/detectors/adobeio/adobeio_test.goindex a65d0035f..649096da0 100644--- a/pkg/detectors/adobeio/adobeio_test.go+++ b/pkg/detectors/adobeio/adobeio_test.go@@ -1,121 +1,86 @@-//go:build detectors-// +build detectors-package adobeioimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "zxcv0987mnbv1234poiu6749gtnrfv54/WDcv0981Mn.B"+ invalidPattern = "Rzxc#0987$%bv1234poiu6749gtnrfv54")-func TestAdobeIO_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ADOBEIO_TOKEN")- id := testSecrets.MustGetField("ADOBEIO_PRODUCT")- inactiveSecret := testSecrets.MustGetField("ADOBEIO_INACTIVE")+func TestAdobeIO_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AdobeIO,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("adobe = '%s'", validPattern),+ want: []string{"zxcv0987mnbv1234poiu6749gtnrfv54WDcv0981Mn.B"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AdobeIO,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("adobe keyword is not close to the real id and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("adobe%s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AdobeIO.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AdobeIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/adzuna/adzuna_integration_test.go b/pkg/detectors/adzuna/adzuna_integration_test.gonew file mode 100644index 000000000..f2a1bf630--- /dev/null+++ b/pkg/detectors/adzuna/adzuna_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package adzuna++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAdzuna_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ADZUNA")+ id := testSecrets.MustGetField("ADZUNA_ID")+ inactiveSecret := testSecrets.MustGetField("ADZUNA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Adzuna,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Adzuna,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Adzuna,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Adzuna,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Adzuna.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Adzuna.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/adzuna/adzuna_test.go b/pkg/detectors/adzuna/adzuna_test.goindex f2a1bf630..83dd82037 100644--- a/pkg/detectors/adzuna/adzuna_test.go+++ b/pkg/detectors/adzuna/adzuna_test.go@@ -1,167 +1,96 @@-//go:build detectors-// +build detectors-package adzunaimport ("context""fmt"+ "strings""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAdzuna_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ADZUNA")- id := testSecrets.MustGetField("ADZUNA_ID")- inactiveSecret := testSecrets.MustGetField("ADZUNA_INACTIVE")+var (+ validPattern = "asdf0987mnbv1234qsxojb6ygb2wsx0o/qsc0f098"+ invalidPattern = "as#f0987mnbv1234^sxojb6ygb2wsx0o/qsc0f098"+)++func TestAdzuna_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Adzuna,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("adzuna = '%s'", validPattern),+ want: []string{"asdf0987mnbv1234qsxojb6ygb2wsx0oqsc0f098"},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Adzuna,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("adzuna keyword is not close to the real id and secret = '%s'", validPattern),+ want: nil,},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Adzuna,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - only key",+ input: fmt.Sprintf("adzuna %s", strings.Split(validPattern, "/")[0]),+ want: nil,},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Adzuna,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - only id",+ input: fmt.Sprintf("adzuna %s", strings.Split(validPattern, "/")[1]),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("adzuna %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Adzuna.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Adzuna.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/aeroworkflow/aeroworkflow_integration_test.go b/pkg/detectors/aeroworkflow/aeroworkflow_integration_test.gonew file mode 100644index 000000000..3ea3ef555--- /dev/null+++ b/pkg/detectors/aeroworkflow/aeroworkflow_integration_test.go@@ -0,0 +1,165 @@+//go:build detectors+// +build detectors++package aeroworkflow++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAeroworkflow_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AEROWORKFLOW_SECRET")+ id := testSecrets.MustGetField("AEROWORKFLOW_ID")+ inactiveSecret := testSecrets.MustGetField("AEROWORKFLOW_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aeroworkflow,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Aeroworkflow,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Aeroworkflow,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aeroworkflow,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Aeroworkflow.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Aeroworkflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/aeroworkflow/aeroworkflow_test.go b/pkg/detectors/aeroworkflow/aeroworkflow_test.goindex ba46cd93e..2d5dc1a08 100644--- a/pkg/detectors/aeroworkflow/aeroworkflow_test.go+++ b/pkg/detectors/aeroworkflow/aeroworkflow_test.go@@ -1,163 +1,96 @@-//go:build detectors-// +build detectors-package aeroworkflowimport ("context""fmt"+ "strings""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAeroworkflow_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AEROWORKFLOW_SECRET")- id := testSecrets.MustGetField("AEROWORKFLOW_ID")- inactiveSecret := testSecrets.MustGetField("AEROWORKFLOW_INACTIVE")+var (+ validPattern = "qscVgy!WdvG;^#O:*?TG/806445634"+ invalidPattern = "qscVg&!WdvG;^#O:*?TG/8064456A4"+)++func TestAeroWorkflow_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aeroworkflow,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("aeroworkflow = '%s'", validPattern),+ want: []string{"qscVgy!WdvG;^#O:*?TG806445634"},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Aeroworkflow,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("aeroworkflow keyword is not close to the real id and secret = '%s'", validPattern),+ want: nil,},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Aeroworkflow,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - only key",+ input: fmt.Sprintf("aeroworkflow %s", strings.Split(validPattern, "/")[0]),+ want: nil,},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aeroworkflow,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - only id",+ input: fmt.Sprintf("aeroworkflow %s", strings.Split(validPattern, "/")[1]),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("aeroworkflow %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Aeroworkflow.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Aeroworkflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/agora/agora_integration_test.go b/pkg/detectors/agora/agora_integration_test.gonew file mode 100644index 000000000..fbd70bd86--- /dev/null+++ b/pkg/detectors/agora/agora_integration_test.go@@ -0,0 +1,192 @@+//go:build detectors+// +build detectors++package agora++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAgora_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("AGORA")+ secret := testSecrets.MustGetField("AGORA_SECRET")+ inactiveSecret := testSecrets.MustGetField("AGORA_SECRET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: true,+ },+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r, r, r, r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r, r, r, r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but not valid ", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Agora,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/agora/agora_test.go b/pkg/detectors/agora/agora_test.goindex db0f64ac9..4a16637fe 100644--- a/pkg/detectors/agora/agora_test.go+++ b/pkg/detectors/agora/agora_test.go@@ -1,191 +1,96 @@-//go:build detectors-// +build detectors-package agoraimport ("context""fmt"+ "strings""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "asdf0987mnbv1234qsxojb6ygb2wsx0o/beqr7215fr4g6bfjkmnvxrtygb2wsxap"+ invalidPattern = "asdf0987mNbv1234qsxojb6ygb2w$x0o/beqr7215fr4g6bfjkmnVxrtygb2wsxap")-func TestAgora_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("AGORA")- secret := testSecrets.MustGetField("AGORA_SECRET")- inactiveSecret := testSecrets.MustGetField("AGORA_SECRET_INACTIVE")+func TestAgora_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: true,- },- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("agora = '%s'", validPattern),+ want: []string{"asdf0987mnbv1234qsxojb6ygb2wsx0oasdf0987mnbv1234qsxojb6ygb2wsx0o"},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r, r, r, r}- }(),- wantErr: false,+ name: "valid pattern - out of prefix range",+ input: fmt.Sprintf("agora keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r, r, r, r}- }(),- wantErr: false,+ name: "valid pattern - only key",+ input: fmt.Sprintf("agora %s", strings.Split(validPattern, "/")[0]),+ want: []string{"asdf0987mnbv1234qsxojb6ygb2wsx0oasdf0987mnbv1234qsxojb6ygb2wsx0o"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but not valid ", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Agora,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - only secret",+ input: fmt.Sprintf("agora %s", strings.Split(validPattern, "/")[1]),+ want: []string{"beqr7215fr4g6bfjkmnvxrtygb2wsxapbeqr7215fr4g6bfjkmnvxrtygb2wsxap"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("agora %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/aha/aha_integration_test.go b/pkg/detectors/aha/aha_integration_test.gonew file mode 100644index 000000000..548beb044--- /dev/null+++ b/pkg/detectors/aha/aha_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package aha++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAha_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ domain := testSecrets.MustGetField("AHA_DOMAIN")+ secret := testSecrets.MustGetField("AHA_SECRET")+ inactiveSecret := testSecrets.MustGetField("AHA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{client: common.ConstantResponseHttpClient(200, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aha,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Aha,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Aha,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid domain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aha,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Aha.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Aha.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/aha/aha_test.go b/pkg/detectors/aha/aha_test.goindex 5e3a93784..d68de04b7 100644--- a/pkg/detectors/aha/aha_test.go+++ b/pkg/detectors/aha/aha_test.go@@ -1,167 +1,104 @@-//go:build detectors-// +build detectors-package ahaimport ("context""fmt"+ "strings""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff/example.aha.io"+ invalidPattern = "00112233445566778899aabbCC$%eeff00112233445566778899aabbccddeeff/example.fake.io")-func TestAha_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- domain := testSecrets.MustGetField("AHA_DOMAIN")- secret := testSecrets.MustGetField("AHA_SECRET")- inactiveSecret := testSecrets.MustGetField("AHA_INACTIVE")+func TestAha_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})++ key := strings.Split(validPattern, "/")[0]+ url := strings.Split(validPattern, "/")[1]- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{client: common.ConstantResponseHttpClient(200, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aha,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("aha.io = '%s'", validPattern),+ want: []string{key},},{- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Aha,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - detect URL far away from keyword",+ input: fmt.Sprintf("aha.io = '%s\n URL is not close to the keyword but should be detected %s'", key, url),+ want: []string{key},},{- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Aha,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("aha.io keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,+ },+ {+ name: "valid pattern - only key",+ input: fmt.Sprintf("aha.io %s", key),+ want: []string{key},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid domain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aha,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - only URL",+ input: fmt.Sprintf("aha.io %s", url),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("aha.io %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Aha.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Aha.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_integration_test.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_integration_test.gonew file mode 100644index 000000000..1dc9cc5ec--- /dev/null+++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package airbrakeprojectkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAirbrakeProjectKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_TOKEN")+ id := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_ID")+ inactiveSecret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but verified", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AirbrakeProjectKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AirbrakeProjectKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.goindex 3ce173571..7c70bb9c3 100644--- a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go+++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go@@ -1,121 +1,97 @@-//go:build detectors-// +build detectors-package airbrakeprojectkeyimport ("context""fmt"+ "strings""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "qwmnerBv56zxpocvkjqr78afvYUx90Op/451298"+ invalidPattern = "qwmnerBv56zxpocvkjqr78afvYU$90Op/4512987")-func TestAirbrakeProjectKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_TOKEN")- id := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_ID")- inactiveSecret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_INACTIVE")+func TestAirBrakeProjectKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but verified", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("airbrake = '%s'", validPattern),+ want: []string{"qwmnerBv56zxpocvkjqr78afvYUx90Op451298"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("airbrake keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - only key",+ input: fmt.Sprintf("airbrake %s", strings.Split(validPattern, "/")[0]),+ want: nil,+ },+ {+ name: "valid pattern - only ID",+ input: fmt.Sprintf("airbrake %s", strings.Split(validPattern, "/")[0]),+ want: nil,+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("airbrake = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AirbrakeProjectKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AirbrakeProjectKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_integration_test.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_integration_test.gonew file mode 100644index 000000000..7e0f5fabc--- /dev/null+++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package airbrakeuserkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAirbrakeUserKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AIRBRAKEUSERKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("AIRBRAKEUSERKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirbrakeUserKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirbrakeUserKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AirbrakeUserKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AirbrakeUserKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.goindex 462603268..b41696885 100644--- a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go+++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package airbrakeuserkeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "qsCGuilpkk2ngrsz75wtYqsCGuilpkk2ngrsz75w"+ invalidPattern = "Qs%CGuil#pkk2ngrsz75wtYqsCGuilpkk2ngrsz75w")-func TestAirbrakeUserKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AIRBRAKEUSERKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("AIRBRAKEUSERKEY_INACTIVE")+func TestAirBrakeUserKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirbrakeUserKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("airbrake = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirbrakeUserKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("airbrake keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("airbrake = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AirbrakeUserKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AirbrakeUserKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/airship/airship.go b/pkg/detectors/airship/airship.goindex 5c543df08..57eabd568 100644--- a/pkg/detectors/airship/airship.go+++ b/pkg/detectors/airship/airship.go@@ -21,7 +21,7 @@ var (client = common.SaneHttpClient()// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"airship"}) + `\b([0-9Aa-zA-Z]{91})\b`)+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"airship"}) + `\b([0-9a-zA-Z]{91})\b`))// Keywords are used for efficiently pre-filtering chunks.diff --git a/pkg/detectors/airship/airship_integration_test.go b/pkg/detectors/airship/airship_integration_test.gonew file mode 100644index 000000000..8969b280b--- /dev/null+++ b/pkg/detectors/airship/airship_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package airship++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAirship_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AIRSHIP")+ inactiveSecret := testSecrets.MustGetField("AIRSHIP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airship secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Airship,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airship secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Airship,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Airship.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Airship.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/airship/airship_test.go b/pkg/detectors/airship/airship_test.goindex 75f3263db..dc354533b 100644--- a/pkg/detectors/airship/airship_test.go+++ b/pkg/detectors/airship/airship_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package airshipimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "123abcdef456ghijkl789mnopqr012stuvwx3455123abcdef456ghijkl789mnopqr012stuvwx3455trdfrq95acr"+ invalidPattern = "123#abcdef456gh$ijkl789mnopqr012stuvwX3455123abcdef456ghijkl789mnopqr012stuvwx3455trdfrq")-func TestAirship_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AIRSHIP")- inactiveSecret := testSecrets.MustGetField("AIRSHIP_INACTIVE")+func TestAirship_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airship secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Airship,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("airship = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airship secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Airship,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("airship keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("airship = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Airship.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Airship.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/airtableapikey/airtableapikey.go b/pkg/detectors/airtableapikey/airtableapikey.goindex 0938bba0b..bff9284d8 100644--- a/pkg/detectors/airtableapikey/airtableapikey.go+++ b/pkg/detectors/airtableapikey/airtableapikey.go@@ -13,7 +13,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -51,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif len(keyMatches) == 0 {keyMatches = personalKeyMatches-}for _, keyMatch := range keyMatches {diff --git a/pkg/detectors/airtableapikey/airtableapikey_integration_test.go b/pkg/detectors/airtableapikey/airtableapikey_integration_test.gonew file mode 100644index 000000000..ba75a3864--- /dev/null+++ b/pkg/detectors/airtableapikey/airtableapikey_integration_test.go@@ -0,0 +1,125 @@+//go:build detectors+// +build detectors++package airtableapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAirtableApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY")+ inactiveSecret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY_INACTIVE")+ app := testSecrets.MustGetField("AIRTABLEAPIKEY_APP")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable https://api.airtable.com/v0/%s/Projects", secret, app)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirtableApiKey,+ Redacted: app,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable %s", inactiveSecret, app)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirtableApiKey,+ Redacted: app,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AirtableApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/airtableapikey/airtableapikey_test.go b/pkg/detectors/airtableapikey/airtableapikey_test.goindex 6c3e962ed..a9ddfa89a 100644--- a/pkg/detectors/airtableapikey/airtableapikey_test.go+++ b/pkg/detectors/airtableapikey/airtableapikey_test.go@@ -1,125 +1,87 @@-//go:build detectors-// +build detectors-package airtableapikeyimport ("context""fmt""testing"- "time"++ "github.com/google/go-cmp/cmp""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "app_pOcv67-Yuztyq / key_Yuztyq-pOcv67"+ validPersonalKeyPattern = "app_pOcv67-Yuztyq / patWtrafrcg64DP0w.AA1AA2BB3CC4DD5EE6FF7GG8HH9II0JJ1KK2LL3MM4NN5OO6PP7QQ8aaaaaaaaaa"+ invalidPattern = "app_pOcv67%Yuztyq/key_Yuztyq*pOcv67")-func TestAirtableApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY")- inactiveSecret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY_INACTIVE")- app := testSecrets.MustGetField("AIRTABLEAPIKEY_APP")+func TestAirTableApiKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable https://api.airtable.com/v0/%s/Projects", secret, app)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirtableApiKey,- Redacted: app,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with key",+ input: fmt.Sprintf("airtable secrets: %s", validPattern),+ want: []string{"key_Yuztyq-pOcv67app_pOcv67-Yuztyq"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable %s", inactiveSecret, app)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirtableApiKey,- Redacted: app,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with personal key",+ input: fmt.Sprintf("airtable secrets: %s", validPersonalKeyPattern),+ want: []string{"patWtrafrcg64DP0w.AA1AA2BB3CC4DD5EE6FF7GG8HH9II0JJ1KK2LL3MM4NN5OO6PP7QQ8aaaaaaaaaaapp_pOcv67-Yuztyq"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("airtable secrets: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AirtableApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/airvisual/airvisual_integration_test.go b/pkg/detectors/airvisual/airvisual_integration_test.gonew file mode 100644index 000000000..74af10ded--- /dev/null+++ b/pkg/detectors/airvisual/airvisual_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package airvisual++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAirVisual_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AIRVISUAL")+ inactiveSecret := testSecrets.MustGetField("AIRVISUAL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airvisual secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirVisual,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a airvisual secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AirVisual,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AirVisual.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AirVisual.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/airvisual/airvisual_test.go b/pkg/detectors/airvisual/airvisual_test.goindex cef7e9488..b4d585384 100644--- a/pkg/detectors/airvisual/airvisual_test.go+++ b/pkg/detectors/airvisual/airvisual_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package airvisualimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "qscgyygcsq-wdvvok7slklklaasnd8afafxd"+ invalidPattern = "wdvvok7slklklaasnd8afafxd")-func TestAirVisual_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AIRVISUAL")- inactiveSecret := testSecrets.MustGetField("AIRVISUAL_INACTIVE")+func TestAirVisual_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airvisual secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirVisual,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("airvisual = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a airvisual secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AirVisual,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("airvisual keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("airvisual key: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AirVisual.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AirVisual.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/aiven/aiven_integration_test.go b/pkg/detectors/aiven/aiven_integration_test.gonew file mode 100644index 000000000..5939a1726--- /dev/null+++ b/pkg/detectors/aiven/aiven_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package aiven++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAiven_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AIVEN")+ inactiveSecret := testSecrets.MustGetField("AIVEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aiven secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aiven,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aiven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aiven,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Aiven.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Aiven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/aiven/aiven_test.go b/pkg/detectors/aiven/aiven_test.goindex 69ba404ca..f6447a02f 100644--- a/pkg/detectors/aiven/aiven_test.go+++ b/pkg/detectors/aiven/aiven_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package aivenimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef12`+ invalidPattern = `ABCdef123`)-func TestAiven_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AIVEN")- inactiveSecret := testSecrets.MustGetField("AIVEN_INACTIVE")+func TestAiven_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aiven secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aiven,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("aiven = '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aiven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aiven,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("aiven keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("aiven = '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Aiven.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Aiven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/alconost/alconost_integration_test.go b/pkg/detectors/alconost/alconost_integration_test.gonew file mode 100644index 000000000..6e3fda8fd--- /dev/null+++ b/pkg/detectors/alconost/alconost_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package alconost++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAlconost_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALCONOST")+ inactiveSecret := testSecrets.MustGetField("ALCONOST_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alconost secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alconost,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alconost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alconost,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Alconost.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Alconost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/alconost/alconost_test.go b/pkg/detectors/alconost/alconost_test.goindex 96551e827..0c4cc1f0e 100644--- a/pkg/detectors/alconost/alconost_test.go+++ b/pkg/detectors/alconost/alconost_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package alconostimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "wdvnousa87acfxp9ioasrea4tbeasrfa"+ invalidPattern = "wdvnousa87acfxp9ioasr$a4tBeasrfa")-func TestAlconost_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALCONOST")- inactiveSecret := testSecrets.MustGetField("ALCONOST_INACTIVE")+func TestAlconost_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alconost secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Alconost,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("alconost: '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alconost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Alconost,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("alconost keyword is not close to the real key in the data = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("alconost: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Alconost.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Alconost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/alegra/alegra.go b/pkg/detectors/alegra/alegra.goindex 065b44233..1fcbfcb2b 100644--- a/pkg/detectors/alegra/alegra.go+++ b/pkg/detectors/alegra/alegra.go@@ -12,7 +12,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -24,7 +24,7 @@ var (// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-z0-9-]{20})\b`)- idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9.-@]{25,30})\b`)+ idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9\.\-\@]{25,30})\b`))// Keywords are used for efficiently pre-filtering chunks.diff --git a/pkg/detectors/alegra/alegra_integration_test.go b/pkg/detectors/alegra/alegra_integration_test.gonew file mode 100644index 000000000..19913d9c8--- /dev/null+++ b/pkg/detectors/alegra/alegra_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package alegra++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAlegra_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALEGRA")+ id := testSecrets.MustGetField("ACCOUNT_USER")+ inactiveSecret := testSecrets.MustGetField("ALEGRA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alegra,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alegra,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Alegra.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/alegra/alegra_test.go b/pkg/detectors/alegra/alegra_test.goindex 31fe6fb58..d2eb6e346 100644--- a/pkg/detectors/alegra/alegra_test.go+++ b/pkg/detectors/alegra/alegra_test.go@@ -1,121 +1,92 @@-//go:build detectors-// +build detectors-package alegraimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "wdvn-usa87a-fxp9ioas/[email protected]"+ validSpecialCharPattern = "wdvn-usa87a-fxp9ioas / [email protected]"+ invalidPattern = "wdvn-usa87a-fxp9ioas/[email protected]")-func TestAlegra_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALEGRA")- id := testSecrets.MustGetField("ACCOUNT_USER")- inactiveSecret := testSecrets.MustGetField("ALEGRA_INACTIVE")+func TestAlegra_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Alegra,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("alegra: '%s'", validPattern),+ want: []string{"[email protected]"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Alegra,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with special characters",+ input: fmt.Sprintf("alegra: '%s'", validSpecialCharPattern),+ want: []string{"[email protected]"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("alegra keyword is not close to the real key and id = '%s'", validPattern),+ want: nil,+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("alegra: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Alegra.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/aletheiaapi/aletheiaapi_integration_test.go b/pkg/detectors/aletheiaapi/aletheiaapi_integration_test.gonew file mode 100644index 000000000..32b39a124--- /dev/null+++ b/pkg/detectors/aletheiaapi/aletheiaapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package aletheiaapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAletheiaApi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALETHEIAAPI")+ inactiveSecret := testSecrets.MustGetField("ALETHEIAAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AletheiaApi,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AletheiaApi,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AletheiaApi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AletheiaApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/aletheiaapi/aletheiaapi_test.go b/pkg/detectors/aletheiaapi/aletheiaapi_test.goindex 32b39a124..e651d037f 100644--- a/pkg/detectors/aletheiaapi/aletheiaapi_test.go+++ b/pkg/detectors/aletheiaapi/aletheiaapi_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package aletheiaapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "TEST01KEYPATTERN2024ALETHEIAAPIN"+ invalidPattern = "TEST01kEYPATTERN2024ALETHEIAAPIN")-func TestAletheiaApi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALETHEIAAPI")- inactiveSecret := testSecrets.MustGetField("ALETHEIAAPI_INACTIVE")+func TestAleTheIaAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AletheiaApi,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("aletheiaapi: '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AletheiaApi,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("aletheiaapi keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("aletheiaapi: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AletheiaApi.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AletheiaApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.gonew file mode 100644index 000000000..3a6985580--- /dev/null+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package algoliaadminkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAlgoliaAdminKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALGOLIAADMINKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("ALGOLIAADMINKEY_INACTIVE")+ id := testSecrets.MustGetField("ALGOLIAADMINKEY_APPID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AlgoliaAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AlgoliaAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.goindex 3a6985580..a433d34a7 100644--- a/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go@@ -1,121 +1,86 @@-//go:build detectors-// +build detectors-package algoliaadminkeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "AlgoliaAdminKey010TESTINGPlmiOPC/ADMIN01KEY"+ invalidPattern = "AlgoliaAdminN$TKey010TESTINGPlmiOPC/NonAdmin01ID")-func TestAlgoliaAdminKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALGOLIAADMINKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("ALGOLIAADMINKEY_INACTIVE")- id := testSecrets.MustGetField("ALGOLIAADMINKEY_APPID")+func TestAlgoliaAdminKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("algolia: '%s'", validPattern),+ want: []string{"AlgoliaAdminKey010TESTINGPlmiOPCADMIN01KEY"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("algolia keyword is not close to the real key and secret = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("algolia: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AlgoliaAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AlgoliaAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/alibaba/alibaba_integration_test.go b/pkg/detectors/alibaba/alibaba_integration_test.gonew file mode 100644index 000000000..31b81881e--- /dev/null+++ b/pkg/detectors/alibaba/alibaba_integration_test.go@@ -0,0 +1,186 @@+//go:build detectors+// +build detectors++package alibaba++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAlibaba_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALIBABA_SECRET")+ inactiveSecret := testSecrets.MustGetField("ALIBABA_SECRET_INACTIVE")+ id := testSecrets.MustGetField("ALIBABA_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to broken json",+ s: Scanner{client: common.ConstantResponseHttpClient(418, "{")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected EOF"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Alibaba,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/alibaba/alibaba_test.go b/pkg/detectors/alibaba/alibaba_test.goindex 31b81881e..8edb6f4cc 100644--- a/pkg/detectors/alibaba/alibaba_test.go+++ b/pkg/detectors/alibaba/alibaba_test.go@@ -1,185 +1,85 @@-//go:build detectors-// +build detectors-package alibabaimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "abcDEF123ghiJKL456mnoPQR789std/LTAI123ABCdef456ghijkLMN"+ invalidPattern = "abcDEF123ghiJKL456m$oPQR789/123ABCdef456ghijkLMN;")-func TestAlibaba_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALIBABA_SECRET")- inactiveSecret := testSecrets.MustGetField("ALIBABA_SECRET_INACTIVE")- id := testSecrets.MustGetField("ALIBABA_ID")+func TestAliBaba_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Alibaba,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Alibaba,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Alibaba,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, real secrets, verification error due to broken json",- s: Scanner{client: common.ConstantResponseHttpClient(418, "{")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Alibaba,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected EOF"))- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("LTAI: '%s'", validPattern),+ want: []string{"abcDEF123ghiJKL456mnoPQR789stdLTAI123ABCdef456ghijkLMN"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Alibaba,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore special characters at end",+ input: fmt.Sprintf("LTAI: '%s\"''", validPattern),+ want: []string{"abcDEF123ghiJKL456mnoPQR789stdLTAI123ABCdef456ghijkLMN"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("LTAI: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/alienvault/alienvault_integration_test.go b/pkg/detectors/alienvault/alienvault_integration_test.gonew file mode 100644index 000000000..ab9e035bd--- /dev/null+++ b/pkg/detectors/alienvault/alienvault_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package alienvault++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAlienVault_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALIENVAULT")+ inactiveSecret := testSecrets.MustGetField("ALIENVAULT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alienvault secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AlienVault,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a alienvault secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AlienVault,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AlienVault.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AlienVault.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/alienvault/alienvault_test.go b/pkg/detectors/alienvault/alienvault_test.goindex afff2582a..514c5de34 100644--- a/pkg/detectors/alienvault/alienvault_test.go+++ b/pkg/detectors/alienvault/alienvault_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package alienvaultimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "a3b9f8c1e2d4f5b6c7d8e9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0"+ invalidPattern = "A$3b9f8c1e2d4f5b6c7d8e9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d")-func TestAlienVault_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALIENVAULT")- inactiveSecret := testSecrets.MustGetField("ALIENVAULT_INACTIVE")+func TestAlienVault_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alienvault secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AlienVault,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("alienvault: '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a alienvault secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AlienVault,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("alienvault keyword is not close to the real key in the data\n = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("alienvault: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AlienVault.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AlienVault.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}