Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Added pattern unit tests for detectors starting with the letters r through s (#3752)
Nabeel Alame6be459eb411
diff --git a/pkg/detectors/rabbitmq/rabbitmq_integration_test.go b/pkg/detectors/rabbitmq/rabbitmq_integration_test.gonew file mode 100644index 000000000..f71148ff1--- /dev/null+++ b/pkg/detectors/rabbitmq/rabbitmq_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package rabbitmq++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRabbitMQ_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RABBITMQ")+ inactiveSecret := testSecrets.MustGetField("RABBITMQ_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RabbitMQ,+ Verified: true,+ Redacted: "amqp://rabbit123:*******************@127.0.0.1",+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RabbitMQ,+ Verified: false,+ Redacted: "amqp://rabbituser:*******************@127.0.0.1",+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RabbitMQ.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RabbitMQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rabbitmq/rabbitmq_test.go b/pkg/detectors/rabbitmq/rabbitmq_test.goindex 77a5ffcd6..d21856fd9 100644--- a/pkg/detectors/rabbitmq/rabbitmq_test.go+++ b/pkg/detectors/rabbitmq/rabbitmq_test.go@@ -1,122 +1,86 @@-//go:build detectors-// +build detectors-package rabbitmqimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRabbitMQ_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RABBITMQ")- inactiveSecret := testSecrets.MustGetField("RABBITMQ_INACTIVE")+var (+ validPattern = "amqps://user123:[email protected]:5672/vhost789"+ invalidPattern = "amqps://user123:pass456@?abbitmq-host.pattern.com:5672/vhost789"+ keyword = "rabbitmq"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRabbitMQ_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RabbitMQ,- Verified: true,- Redacted: "amqp://rabbit123:*******************@127.0.0.1",- },- },- wantErr: false,+ name: "valid pattern - with keyword rabbitmq",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RabbitMQ,- Verified: false,- Redacted: "amqp://rabbituser:*******************@127.0.0.1",- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RabbitMQ.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RabbitMQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/ramp/ramp_integration_test.go b/pkg/detectors/ramp/ramp_integration_test.gonew file mode 100644index 000000000..375ff3c51--- /dev/null+++ b/pkg/detectors/ramp/ramp_integration_test.go@@ -0,0 +1,163 @@+//go:build detectors+// +build detectors++package ramp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRamp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ id := testSecrets.MustGetField("RAMP")+ secret := testSecrets.MustGetField("RAMP_SECRET")+ inactiveId := testSecrets.MustGetField("RAMP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ramp client id %s and a ramp secret %s within", id, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ramp,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within but not valid", inactiveId, secret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ramp,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within", id, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ramp,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ramp id %s and secret %s within", id, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ramp,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Ramp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Ramp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/ramp/ramp_test.go b/pkg/detectors/ramp/ramp_test.goindex 7989a7509..8969666fc 100644--- a/pkg/detectors/ramp/ramp_test.go+++ b/pkg/detectors/ramp/ramp_test.go@@ -1,163 +1,82 @@-//go:build detectors-// +build detectors-package rampimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRamp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }-- id := testSecrets.MustGetField("RAMP")- secret := testSecrets.MustGetField("RAMP_SECRET")- inactiveId := testSecrets.MustGetField("RAMP_INACTIVE")+var (+ validKey = "ramp_id_tz8O3QzUBiPcAejdGSIvfcvOxbk1wVfnUEIsieV3"+ invalidKey = "ramp_id_tz8O3QzUBiPcAejd?SIvfcvOxbk1wVfnUEIsieV3"+ validSecret = "ramp_sec_maFm2aZYzbZOVNbEy4RiuvREnhZ2Xe19E45hl4A3en20WJyX"+ invalidSecret = "ramp_sec_maFm2aZYzbZOVNbEy4R?uvREnhZ2Xe19E45hl4A3en20WJyX"+ keyword = "ramp"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRamp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ramp client id %s and a ramp secret %s within", id, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ramp,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within but not valid", inactiveId, secret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ramp,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within", id, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ramp,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - with keyword ramp",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),+ want: []string{validKey + ":" + validSecret},},{- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ramp id %s and secret %s within", id, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ramp,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Ramp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Ramp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/rapidapi/rapidapi_integration_test.go b/pkg/detectors/rapidapi/rapidapi_integration_test.gonew file mode 100644index 000000000..671894841--- /dev/null+++ b/pkg/detectors/rapidapi/rapidapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rapidapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRapidApi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RAPIDAPI_TOKEN")+ inactiveSecret := testSecrets.MustGetField("RAPIDAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RapidApi,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RapidApi,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RapidApi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RapidApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rapidapi/rapidapi_test.go b/pkg/detectors/rapidapi/rapidapi_test.goindex 3deee3a8a..956821d36 100644--- a/pkg/detectors/rapidapi/rapidapi_test.go+++ b/pkg/detectors/rapidapi/rapidapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package rapidapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRapidApi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RAPIDAPI_TOKEN")- inactiveSecret := testSecrets.MustGetField("RAPIDAPI_INACTIVE")+var (+ validPattern = "HG_Wv9C_fvjMHOUokbSvUCEw3scqZB1Q4LBgw5u8Nb-Khx8HBi"+ invalidPattern = "HG_Wv9C_fvjMHOUokbSvUCEw3?cqZB1Q4LBgw5u8Nb-Khx8HBi"+ keyword = "rapidapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRapidApi_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RapidApi,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rapidapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RapidApi,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RapidApi.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RapidApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/raven/raven_integration_test.go b/pkg/detectors/raven/raven_integration_test.gonew file mode 100644index 000000000..ee8e9a980--- /dev/null+++ b/pkg/detectors/raven/raven_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package raven++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRaven_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RAVEN")+ inactiveSecret := testSecrets.MustGetField("RAVEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a raven secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Raven,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a raven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Raven,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Raven.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Raven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/raven/raven_test.go b/pkg/detectors/raven/raven_test.goindex 0558e15a2..a2089db5a 100644--- a/pkg/detectors/raven/raven_test.go+++ b/pkg/detectors/raven/raven_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package ravenimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRaven_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RAVEN")- inactiveSecret := testSecrets.MustGetField("RAVEN_INACTIVE")+var (+ validPattern = "JJ4ZIC9EJNWVIS86"+ invalidPattern = "?J4ZIC9EJNWVIS86"+ keyword = "raven"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRaven_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a raven secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Raven,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword raven",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a raven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Raven,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Raven.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Raven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rawg/rawg_integration_test.go b/pkg/detectors/rawg/rawg_integration_test.gonew file mode 100644index 000000000..93b4b7752--- /dev/null+++ b/pkg/detectors/rawg/rawg_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rawg++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRawg_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RAWG")+ inactiveSecret := testSecrets.MustGetField("RAWG_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rawg secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rawg,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rawg secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rawg,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Rawg.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Rawg.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rawg/rawg_test.go b/pkg/detectors/rawg/rawg_test.goindex 8fc636833..986c50eee 100644--- a/pkg/detectors/rawg/rawg_test.go+++ b/pkg/detectors/rawg/rawg_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package rawgimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRawg_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RAWG")- inactiveSecret := testSecrets.MustGetField("RAWG_INACTIVE")+var (+ validPattern = "43zgwcor66n9phsxAxxfi7j81fpaecwy"+ invalidPattern = "43zgwcor66n9phsx?xxfi7j81fpaecwy"+ keyword = "rawg"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRawg_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rawg secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rawg,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rawg",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rawg secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rawg,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Rawg.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Rawg.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/razorpay/razorpay_integration_test.go b/pkg/detectors/razorpay/razorpay_integration_test.gonew file mode 100644index 000000000..21703bfff--- /dev/null+++ b/pkg/detectors/razorpay/razorpay_integration_test.go@@ -0,0 +1,109 @@+//go:build detectors+// +build detectors++package razorpay++import (+ "context"+ "fmt"+ "os"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRazorPay_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secretInactive := testSecrets.MustGetField("RAZORPAY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_live_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RazorPay,+ Redacted: "rzp_live_SnDTaP1ncfliDt",+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RazorPay.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ got[i].RawV2 = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/razorpay/razorpay_test.go b/pkg/detectors/razorpay/razorpay_test.goindex 2e39ec160..b7c6a7897 100644--- a/pkg/detectors/razorpay/razorpay_test.go+++ b/pkg/detectors/razorpay/razorpay_test.go@@ -1,107 +1,82 @@-//go:build detectors-// +build detectors-package razorpayimport ("context""fmt"- "os""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRazorPay_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secretInactive := testSecrets.MustGetField("RAZORPAY_INACTIVE")+var (+ validKey = "rzp_live_oE9IEkoNYnmwkG"+ invalidKey = "rzp_live_oE?IEkoNYnmwkG"+ validSecret = "u5FJFQ0KuTe4hD8iwza076e2"+ invalidSecret = "u5FJFQ0KuTe4?D8iwza076e2"+ keyword = "razorpay"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRazorPay_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_live_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RazorPay,- Redacted: "rzp_live_SnDTaP1ncfliDt",- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword razorpay",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),+ want: []string{validKey + validSecret},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RazorPay.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- if os.Getenv("FORCE_PASS_DIFF") == "true" {++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}- got[i].Raw = nil- got[i].RawV2 = nil}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/reachmail/reachmail_integration_test.go b/pkg/detectors/reachmail/reachmail_integration_test.gonew file mode 100644index 000000000..215a22404--- /dev/null+++ b/pkg/detectors/reachmail/reachmail_integration_test.go@@ -0,0 +1,119 @@+//go:build detectors+// +build detectors++package reachmail++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestReachmail_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REACHMAIL")+ inactiveSecret := testSecrets.MustGetField("REACHMAIL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a reachmail secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Reachmail,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a reachmail secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Reachmail,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Reachmail.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Reachmail.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/reachmail/reachmail_test.go b/pkg/detectors/reachmail/reachmail_test.goindex f33fa67e1..e18fd5561 100644--- a/pkg/detectors/reachmail/reachmail_test.go+++ b/pkg/detectors/reachmail/reachmail_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package reachmailimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestReachmail_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REACHMAIL")- inactiveSecret := testSecrets.MustGetField("REACHMAIL_INACTIVE")+var (+ validPattern = "qFcJR3KhMRWcm1y2O7C1_8pr-VQJgQ8vBvVKmctf8JRr5CFfU6VF9Np2ZMZatF7k"+ invalidPattern = "qFcJR3KhMRWcm1y2O7C1_8pr-VQJgQ8v?vVKmctf8JRr5CFfU6VF9Np2ZMZatF7k"+ keyword = "reachmail"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestReachmail_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a reachmail secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Reachmail,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword reachmail",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a reachmail secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Reachmail,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Reachmail.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Reachmail.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/readme/readme_integration_test.go b/pkg/detectors/readme/readme_integration_test.gonew file mode 100644index 000000000..1ef3c1fe1--- /dev/null+++ b/pkg/detectors/readme/readme_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package readme++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestReadMe_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("README")+ inactiveSecret := testSecrets.MustGetField("README_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a readme secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ReadMe,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a readme secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ReadMe,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ReadMe.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ReadMe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/readme/readme_test.go b/pkg/detectors/readme/readme_test.goindex 1648b60f1..d44751c60 100644--- a/pkg/detectors/readme/readme_test.go+++ b/pkg/detectors/readme/readme_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package readmeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestReadMe_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("README")- inactiveSecret := testSecrets.MustGetField("README_INACTIVE")+var (+ validPattern = "rdme_xybq9sovqyexf5wtc437abdsgvw6dfvagdstx3leyr4t2cam7xk0ukv14on7ghun67curt"+ invalidPattern = "rdme_xybq9sovqyexf5wtc437abds?vw6dfvagdstx3leyr4t2cam7xk0ukv14on7ghun67curt"+ keyword = "readme"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestReadMe_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a readme secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ReadMe,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword readme",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a readme secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ReadMe,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ReadMe.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ReadMe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems_integration_test.go b/pkg/detectors/reallysimplesystems/reallysimplesystems_integration_test.gonew file mode 100644index 000000000..197de11b3--- /dev/null+++ b/pkg/detectors/reallysimplesystems/reallysimplesystems_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package reallysimplesystems++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestReallySimpleSystems_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS")+ inactiveSecret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ReallySimpleSystems,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ReallySimpleSystems,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ReallySimpleSystems.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ReallySimpleSystems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.goindex 631bac585..2308b10b0 100644--- a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go+++ b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package reallysimplesystemsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestReallySimpleSystems_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS")- inactiveSecret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS_INACTIVE")+var (+ validPattern = "eyt_8-ytglklqvyjGF6dgbXsCSMRLdryUWQCSQ1n1uSph1Ppl.jMeMx0EkxVLcmbKPTi7kPYKzNa04V96eiJlV.XxtKFDKTJCkcThvKR2x0ctVUL.FrbHVXEx55VZZPD0FKW4onNDO05Ab_9cbnrLTr4d--.eyX.hZw.Q5g-pvyWS5jjDUyiE60WZSgQq4uM8PUVw1-OJegMnGGLEKq48G5X7zgQAcYvUIRQy4Ku.gKQkNFIL4Nlj3ob7Bi4VUDs_DjvtS_Jxho-Hn_hgIhfYQVw19wQCcc8m7v2jCXz2MTi74vpzW7aY2C123i5uFr1KdVQLX.Xlf5ZtuLAsYZ5MsG1G6VYO_t7l6nl8D-Y3fml7uSGPXkeQT7AW1HKOTrBdGj0L_OMMYBaZ4-R9tnUjM7Jf1LExVn2NIQnNGR7Jyl69WigxQGLaUbNvE9DQkVtV4ar0FORi8mrwAQdb5UxZkOxW4me_F189Y-rs0h9q4vrjmM3Fu1Uj_UCrV4IP6AYr4b9yXzgAlcybQSNUISltLwYC.07LKjRy.dLvF-A2b5mIjyKbR3rOUmiXfeAveBRakkEOKSNOSTDVM839O9a-8tqAggj8eDdPunO8Pg6L_Af_eJowZNhtRluVuFDTHuUW.lAG1bqm.ZJQak800cspbDbB8InwOb0V1UvoO.c.ZOAWM3fQVoidNeDbKbWKR3JdE3iWCGsqz89Zdd6LkxwHAd5TRLn.ZVg67d1.g2S3kadXrTZ0qN6o_7apQ38PbGKydnCTLMfdgN6uIPqAtQntsF6YNvNfFHnXgsfhTkVUDJc5g6oQB_jDeGa8rkCmH6_DO6Z1qV04G.fj.QLJIgMhcdGL8Y3NMRv87spZxNWmX1glZ1AYQlWbbqwzAC1v-p0ImXQSuq6G8aIt8e4w7FnOmOn_Q0NwD85JRgdy.W4f.5hX-zLhj1RDqvC1-Ebd2OLDdne0xqMMNe4kGnzQ6dpDtbZXPcim443CyFxaFD_DK5-Iupc032AKV_Sm9eDSlRuAmAQ7LhvypNWi_6e9NuVUEQ5P6HfBe1eXkZ3y5HkZpWbJqJW2D"+ invalidPattern = "ayt_8-ytglklqvyjGF6dgbXsCSMRLdryUWQCSQ1n1uSph1Ppl.jMeMx0EkxVLcmbKPTi7kPYKzNa04V96eiJlV.XxtKFDKTJCkcThvKR2x0ctVUL.FrbHVXEx55VZZPD0FKW4onNDO05Ab_9cbnrLTr4d--.eyX.hZw.Q5g-pvyWS5jjDUyiE60WZSgQq4uM8PUVw1-OJegMnGGLEKq48G5X7zgQAcYvUIRQy4Ku.gKQkNFIL4Nlj3ob7Bi4VUDs_DjvtS_Jxho-Hn_hgIhfYQVw19wQCcc8m7v2jCXz2MTi74vpzW7aY2C123i5uFr1KdVQLX.Xlf5ZtuLAsYZ5MsG1G6VYO_t7l6nl8D-Y3fml7uSGPXkeQT7AW1HKOTrBdGj0L_OMMYBaZ4-R9tnUjM7Jf1LExVn2NIQnNGR7Jyl69WigxQGLaUbNvE9DQkVtV4ar0FORi8mrwAQdb5UxZkOxW4me_F189Y-rs0h9q4vrjmM3Fu1Uj_UCrV4IP6AYr4b9yXzgAlcybQSNUISltLwYC.07LKjRy.dLvF-A2b5mIjyKbR3rOUmiXfeAveBRakkEOKSNOSTDVM839O9a-8tqAggj8eDdPunO8Pg6L_Af_eJowZNhtRluVuFDTHuUW.lAG1bqm.ZJQak800cspbDbB8InwOb0V1UvoO.c.ZOAWM3fQVoidNeDbKbWKR3JdE3iWCGsqz89Zdd6LkxwHAd5TRLn.ZVg67d1.g2S3kadXrTZ0qN6o_7apQ38PbGKydnCTLMfdgN6uIPqAtQntsF6YNvNfFHnXgsfhTkVUDJc5g6oQB_jDeGa8rkCmH6_DO6Z1qV04G.fj.QLJIgMhcdGL8Y3NMRv87spZxNWmX1glZ1AYQlWbbqwzAC1v-p0ImXQSuq6G8aIt8e4w7FnOmOn_Q0NwD85JRgdy.W4f.5hX-zLhj1RDqvC1-Ebd2OLDdne0xqMMNe4kGnzQ6dpDtbZXPcim443CyFxaFD_DK5-Iupc032AKV_Sm9eDSlRuAmAQ7LhvypNWi_6e9NuVUEQ5P6HfBe1eXkZ3y5HkZpWbJqJW2D"+ keyword = "reallysimplesystems"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestReallySimpleSystems_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ReallySimpleSystems,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword reallysimplesystems",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ReallySimpleSystems,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ReallySimpleSystems.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ReallySimpleSystems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rebrandly/rebrandly_integration_test.go b/pkg/detectors/rebrandly/rebrandly_integration_test.gonew file mode 100644index 000000000..a9abde619--- /dev/null+++ b/pkg/detectors/rebrandly/rebrandly_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rebrandly++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRebrandly_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REBRANDLY")+ inactiveSecret := testSecrets.MustGetField("REBRANDLY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rebrandly,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rebrandly,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Rebrandly.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Rebrandly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rebrandly/rebrandly_test.go b/pkg/detectors/rebrandly/rebrandly_test.goindex f1b85b2ad..5ac8e8ce5 100644--- a/pkg/detectors/rebrandly/rebrandly_test.go+++ b/pkg/detectors/rebrandly/rebrandly_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package rebrandlyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRebrandly_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REBRANDLY")- inactiveSecret := testSecrets.MustGetField("REBRANDLY_INACTIVE")+var (+ validPattern = "_dd9gKTi7gDhQYowXu5efwPLKkAAQ4qP"+ invalidPattern = "_dd9gKTi7gDhQYow?u5efwPLKkAAQ4qP"+ keyword = "rebrandly"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRebrandly_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rebrandly,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rebrandly",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rebrandly,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Rebrandly.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Rebrandly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rechargepayments/rechargepayments_integration_test.go b/pkg/detectors/rechargepayments/rechargepayments_integration_test.gonew file mode 100644index 000000000..57492ecee--- /dev/null+++ b/pkg/detectors/rechargepayments/rechargepayments_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rechargepayments++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRechargePayments_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RECHARGEPAYMENTS")+ inactiveSecret := testSecrets.MustGetField("RECHARGEPAYMENTS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RechargePayments,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RechargePayments,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RechargePayments.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RechargePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rechargepayments/rechargepayments_test.go b/pkg/detectors/rechargepayments/rechargepayments_test.goindex 919b73909..d93028af5 100644--- a/pkg/detectors/rechargepayments/rechargepayments_test.go+++ b/pkg/detectors/rechargepayments/rechargepayments_test.go@@ -1,120 +1,85 @@-//go:build detectors-// +build detectors-package rechargepaymentsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRechargePayments_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RECHARGEPAYMENTS")- inactiveSecret := testSecrets.MustGetField("RECHARGEPAYMENTS_INACTIVE")+var (+ validApiKey = "sk_test_3x3_2fbA8B8f75b3f74D3caEF6ec023c3AAbbe48Edcc2393dc343688Dfa8bbcb141f"+ invalidApiKey = "sk_test_3x3_?fbA8B8f75b3f74D3caEF6ec023c3AAbbe48Edcc2393dc343688Dfa8bbcb141f"+ validOldApiSha = "09ECafFABafCeDec29b51B425E1Ed5BC70c6C88c8CeddDe0ba7ab0Af"+ invalidOldApiSha = "0?ECafFABafCeDec29b51B425E1Ed5BC70c6C88c8CeddDe0ba7ab0Af"+ validOldApiMd5 = "FCd1AAaD6EF3D4bCFdaB9c831cBc0d57"+ invalidOldApiMd5 = "F?d1AAaD6EF3D4bCFdaB9c831cBc0d57"+ keyword = "rechargepayments"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRechargePayments_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RechargePayments,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rechargepayments",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validApiKey, keyword, validOldApiSha, keyword, validOldApiMd5),+ want: []string{validApiKey, validOldApiSha, validOldApiMd5},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RechargePayments,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidApiKey, keyword, invalidOldApiSha, keyword, invalidOldApiMd5),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RechargePayments.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RechargePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/redis/redis_integration_test.go b/pkg/detectors/redis/redis_integration_test.goindex caebf3bb8..85ff1b8ff 100644--- a/pkg/detectors/redis/redis_integration_test.go+++ b/pkg/detectors/redis/redis_integration_test.go@@ -154,3 +154,78 @@ func TestRedisIntegration_FromChunk(t *testing.T) {})}}++func TestURI_FromChunk(t *testing.T) {+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "bad scheme",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("file://user:[email protected]:123/wh/at/ever"),+ verify: true,+ },+ wantErr: false,+ },+ {+ name: "unverified Redis",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("redis://user:[email protected]"),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Redis,+ Verified: false,+ Redacted: "redis://user:*******@redis.com",+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.goindex 92b58dece..fccd171ab 100644--- a/pkg/detectors/redis/redis_test.go+++ b/pkg/detectors/redis/redis_test.go@@ -1,88 +1,86 @@-//go:build detectors-// +build detectors-package redisimport ("context"+ "fmt""testing"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestURI_FromChunk(t *testing.T) {- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validKey = "redis://1SHPg6BjyS:$YYbMDot25P@PvfdX9VshFjW"+ invalidKey = "redis://1SHPg6BjySX$YYbMDot25P@PvfdX9VshFjW"+ validDomain = "QZc67koZ6rUV.redis.cache.windows.net:6380"+ invalidDomain = "QZc67koZ6rUV.redis.cache.windows.net:6381"+ password = "Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS"+ validAzureRedis = validDomain + ",password=" + password + ",ssl=True,abortConnect=False"+ invalidAzureRedis = invalidDomain + ",password=" + password + ",ssl=False,abortConnect=True"+ keyword = "redis"+)++func TestRedisIntegration_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "bad scheme",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("file://user:[email protected]:123/wh/at/ever"),- verify: true,- },- wantErr: false,+ name: "valid pattern - with keyword redis",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validAzureRedis),+ want: []string{"rediss://:" + password + "@" + validDomain},},{- name: "unverified Redis",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("redis://user:[email protected]"),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Redis,- Verified: false,- Redacted: "redis://user:*******@redis.com",- },- },- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidAzureRedis),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/refiner/refiner_integration_test.go b/pkg/detectors/refiner/refiner_integration_test.gonew file mode 100644index 000000000..6bc0969cb--- /dev/null+++ b/pkg/detectors/refiner/refiner_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package refiner++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRefiner_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REFINER")+ inactiveSecret := testSecrets.MustGetField("REFINER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a refiner secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Refiner,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a refiner secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Refiner,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Refiner.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Refiner.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/refiner/refiner_test.go b/pkg/detectors/refiner/refiner_test.goindex 68a9207de..fc1e9b41b 100644--- a/pkg/detectors/refiner/refiner_test.go+++ b/pkg/detectors/refiner/refiner_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package refinerimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRefiner_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REFINER")- inactiveSecret := testSecrets.MustGetField("REFINER_INACTIVE")+var (+ validPattern = "2341fb6f-71d4-dbbc-1317-98c28ceb5837"+ invalidPattern = "2341fb6f?71d4-dbbc-1317-98c28ceb5837"+ keyword = "refiner"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRefiner_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a refiner secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Refiner,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword refiner",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a refiner secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Refiner,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Refiner.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Refiner.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rentman/rentman_integration_test.go b/pkg/detectors/rentman/rentman_integration_test.gonew file mode 100644index 000000000..b7a04ede9--- /dev/null+++ b/pkg/detectors/rentman/rentman_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rentman++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRentman_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RENTMAN")+ inactiveSecret := testSecrets.MustGetField("RENTMAN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rentman secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rentman,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rentman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rentman,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Rentman.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Rentman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rentman/rentman_test.go b/pkg/detectors/rentman/rentman_test.goindex 5bd678616..1ae29bdeb 100644--- a/pkg/detectors/rentman/rentman_test.go+++ b/pkg/detectors/rentman/rentman_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package rentmanimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRentman_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RENTMAN")- inactiveSecret := testSecrets.MustGetField("RENTMAN_INACTIVE")+var (+ validPattern = "eytuZDhcszM3EZO7iPeRfJfZMoRMLXe7qoNV.eyA3uoaQ3ttxNcElupuqjIGN2K-bLTq_RVtB6w6ySxFCOttLniQBf84yvwaBBjw4W1veuikC2uWzoW5Gtw-CpEC-.35zBGkn7t7G19FduNmXTy1r_8bXME-VD5xIflYzioiZvyRLfy685qs8TQDIJV7zLi98cPLrLTE_yXwmppu9o88_OWPzh9oAqmVGqvX5Z_1RXqd6e5MsaTFx22tqKw14njwD.qvyujp.NR6QOQmyREUt2XnUtqdupyGX"+ invalidPattern = "aytuZDhcszM3EZO7iPeRfJfZMoRMLXe7qoNV.eyA3uoaQ3ttxNcElupuqjIGN2K-bLTq_RVtB6w6ySxFCOttLniQBf84yvwaBBjw4W1veuikC2uWzoW5Gtw-CpEC-.35zBGkn7t7G19FduNmXTy1r_8bXME-VD5xIflYzioiZvyRLfy685qs8TQDIJV7zLi98cPLrLTE_yXwmppu9o88_OWPzh9oAqmVGqvX5Z_1RXqd6e5MsaTFx22tqKw14njwD.qvyujp.NR6QOQmyREUt2XnUtqdupyG?"+ keyword = "rentman"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRentman_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rentman secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rentman,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rentman",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rentman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rentman,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Rentman.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Rentman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/repairshopr/repairshopr_integration_test.go b/pkg/detectors/repairshopr/repairshopr_integration_test.gonew file mode 100644index 000000000..a6c59bddb--- /dev/null+++ b/pkg/detectors/repairshopr/repairshopr_integration_test.go@@ -0,0 +1,118 @@+//go:build detectors+// +build detectors++package repairshopr++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRepairshopr_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REPAIRSHOPR")+ domain := testSecrets.MustGetField("REPAIRSHOPR_DOMAIN")+ inactiveSecret := testSecrets.MustGetField("REPAIRSHOPR_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Repairshopr,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Repairshopr,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Repairshopr.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Repairshopr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/repairshopr/repairshopr_test.go b/pkg/detectors/repairshopr/repairshopr_test.goindex a6c59bddb..4d18e65ea 100644--- a/pkg/detectors/repairshopr/repairshopr_test.go+++ b/pkg/detectors/repairshopr/repairshopr_test.go@@ -1,117 +1,82 @@-//go:build detectors-// +build detectors-package repairshoprimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRepairshopr_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REPAIRSHOPR")- domain := testSecrets.MustGetField("REPAIRSHOPR_DOMAIN")- inactiveSecret := testSecrets.MustGetField("REPAIRSHOPR_INACTIVE")+var (+ validKey = "sMpDWS14xuMP2E1FMCufXKajIsodvYiuGT9lWll8gYMAYFDRgOD"+ invalidKey = "?MpDWS14xuMP2E1FMCufXKajIsodvYiuGT9lWll8gYMAYFDRgOD"+ validDomain = "uI8mmO1KgWwz"+ invalidDomain = "?I8mmO1KgWw?"+ keyword = "repairshopr"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRepairshopr_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s", secret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Repairshopr,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword repairshopr",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),+ want: []string{validKey, validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Repairshopr,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Repairshopr.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Repairshopr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/replicate/replicate_integration_test.go b/pkg/detectors/replicate/replicate_integration_test.gonew file mode 100644index 000000000..1e9b4c90b--- /dev/null+++ b/pkg/detectors/replicate/replicate_integration_test.go@@ -0,0 +1,127 @@+//go:build detectors+// +build detectors++package replicate++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestReplicate_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REPLICATE")+ inactiveSecret := testSecrets.MustGetField("REPLICATE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a replicate secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Replicate,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a replicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Replicate,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Replicate.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Replicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/replicate/replicate_test.go b/pkg/detectors/replicate/replicate_test.goindex ab597813d..d04c0486b 100644--- a/pkg/detectors/replicate/replicate_test.go+++ b/pkg/detectors/replicate/replicate_test.go@@ -1,127 +1,80 @@-//go:build detectors-// +build detectors-package replicateimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestReplicate_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REPLICATE")- inactiveSecret := testSecrets.MustGetField("REPLICATE_INACTIVE")+var (+ validPattern = "r8_7DTl1j7Nsy6dpc3GGwkd8qsWFl33hy4ICtNzH"+ invalidPattern = "r8_7DTl1j7Nsy6dpc3GG?kd8qsWFl33hy4ICtNzH"+ keyword = "replicate"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestReplicate_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a replicate secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Replicate,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword replicate",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a replicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Replicate,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Replicate.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Replicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/replyio/replyio_integration_test.go b/pkg/detectors/replyio/replyio_integration_test.gonew file mode 100644index 000000000..798fd20cf--- /dev/null+++ b/pkg/detectors/replyio/replyio_integration_test.go@@ -0,0 +1,127 @@+//go:build detectors+// +build detectors++package replyio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestReplyio_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REPLYIO")+ inactiveSecret := testSecrets.MustGetField("REPLYIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a replyio secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ReplyIO,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a replyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ReplyIO,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Replyio.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Replyio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/replyio/replyio_test.go b/pkg/detectors/replyio/replyio_test.goindex 9f01fa023..d1411db91 100644--- a/pkg/detectors/replyio/replyio_test.go+++ b/pkg/detectors/replyio/replyio_test.go@@ -1,127 +1,90 @@-//go:build detectors-// +build detectors-package replyioimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestReplyio_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REPLYIO")- inactiveSecret := testSecrets.MustGetField("REPLYIO_INACTIVE")+var (+ validPattern = "Y8rVcIMii3FFhyQmzEDfttno"+ invalidPattern = "Y8rVcIMii3FF?yQmzEDfttno"+ keyword = "replyio"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestReplyio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a replyio secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ReplyIO,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword replyio",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a replyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ReplyIO,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Replyio.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Replyio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/requestfinance/requestfinance_integration_test.go b/pkg/detectors/requestfinance/requestfinance_integration_test.gonew file mode 100644index 000000000..f6c79fb8f--- /dev/null+++ b/pkg/detectors/requestfinance/requestfinance_integration_test.go@@ -0,0 +1,127 @@+//go:build detectors+// +build detectors++package requestfinance++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRequestfinance_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REQUESTFINANCE")+ inactiveSecret := testSecrets.MustGetField("REQUESTFINANCE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RequestFinance,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RequestFinance,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Requestfinance.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Requestfinance.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/requestfinance/requestfinance_test.go b/pkg/detectors/requestfinance/requestfinance_test.goindex d8dc273f8..4f0b3e044 100644--- a/pkg/detectors/requestfinance/requestfinance_test.go+++ b/pkg/detectors/requestfinance/requestfinance_test.go@@ -1,127 +1,90 @@-//go:build detectors-// +build detectors-package requestfinanceimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRequestfinance_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REQUESTFINANCE")- inactiveSecret := testSecrets.MustGetField("REQUESTFINANCE_INACTIVE")+var (+ validPattern = "IZAT6SA-AJUQES1-BZWZYWJ-VDIXPVG"+ invalidPattern = "IZAT6SA?AJUQES1-BZWZYWJ-VDIXPVG"+ keyword = "requestfinance"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRequestfinance_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RequestFinance,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword requestfinance",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RequestFinance,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Requestfinance.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Requestfinance.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/restpack/restpack_integration_test.go b/pkg/detectors/restpack/restpack_integration_test.gonew file mode 100644index 000000000..05399149d--- /dev/null+++ b/pkg/detectors/restpack/restpack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package restpack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRestpack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RESTPACK")+ inactiveSecret := testSecrets.MustGetField("RESTPACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a restpack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Restpack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a restpack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Restpack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Restpack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Restpack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/restpack/restpack_test.go b/pkg/detectors/restpack/restpack_test.goindex ff3404f3c..e90b8c366 100644--- a/pkg/detectors/restpack/restpack_test.go+++ b/pkg/detectors/restpack/restpack_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package restpackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRestpack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RESTPACK")- inactiveSecret := testSecrets.MustGetField("RESTPACK_INACTIVE")+var (+ validPattern = "RGTuhewdnuYUbL6cFbVTD2DUgVG11qDzuzV1c4gBEUFQaelZ"+ invalidPattern = "RGTuhewdnuYUbL6cFbVTD2DU?VG11qDzuzV1c4gBEUFQaelZ"+ keyword = "restpack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRestpack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a restpack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Restpack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword restpack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a restpack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Restpack,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Restpack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Restpack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_integration_test.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_integration_test.gonew file mode 100644index 000000000..db81cb172--- /dev/null+++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package restpackhtmltopdfapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRestpackHtmlToPdfAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI")+ inactiveSecret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RestpackHtmlToPdfAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RestpackHtmlToPdfAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.goindex 9666074ea..a8e3c4522 100644--- a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go+++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package restpackhtmltopdfapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRestpackHtmlToPdfAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI")- inactiveSecret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI_INACTIVE")+var (+ validPattern = "vVIKB7ypMMXOlpRx2ad1g1t8JmvDBsvQ02LMc1nuNOQYAMEF"+ invalidPattern = "vVIKB7ypMMXOlpRx2ad1g1t8?mvDBsvQ02LMc1nuNOQYAMEF"+ keyword = "restpackhtmltopdfapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRestpackHtmlToPdfAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword restpackhtmltopdfapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RestpackHtmlToPdfAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RestpackHtmlToPdfAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_integration_test.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_integration_test.gonew file mode 100644index 000000000..36a3816a5--- /dev/null+++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package restpackscreenshotapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRestpackScreenshotAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI")+ inactiveSecret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RestpackScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RestpackScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.goindex 36a3816a5..04fda6686 100644--- a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go+++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package restpackscreenshotapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRestpackScreenshotAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI")- inactiveSecret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI_INACTIVE")+var (+ validPattern = "O6xSfj5GgiV3imFS5IU13903DwIfCSnawlBybGQ9sMKg9Ugk"+ invalidPattern = "O6xSfj5GgiV3imFS5IU13903?wIfCSnawlBybGQ9sMKg9Ugk"+ keyword = "restpackscreenshotapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRestpackScreenshotAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword restpackscreenshotapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RestpackScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RestpackScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rev/rev_integration_test.go b/pkg/detectors/rev/rev_integration_test.gonew file mode 100644index 000000000..4b9b34ef5--- /dev/null+++ b/pkg/detectors/rev/rev_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package rev++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRev_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REV")+ inactiveSecret := testSecrets.MustGetField("REV_INACTIVE")+ clientSecret := testSecrets.MustGetField("REV_CLIENT")+ inactiveClientSecret := testSecrets.MustGetField("REV_CLIENT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within", secret, clientSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rev,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within but not valid", inactiveSecret, inactiveClientSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rev,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Rev.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Rev.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rev/rev_test.go b/pkg/detectors/rev/rev_test.goindex 885a1dace..e7b0bc84f 100644--- a/pkg/detectors/rev/rev_test.go+++ b/pkg/detectors/rev/rev_test.go@@ -1,122 +1,83 @@-//go:build detectors-// +build detectors-package revimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRev_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REV")- inactiveSecret := testSecrets.MustGetField("REV_INACTIVE")- clientSecret := testSecrets.MustGetField("REV_CLIENT")- inactiveClientSecret := testSecrets.MustGetField("REV_CLIENT_INACTIVE")+var (+ validUserKey = "T5LJHzWwZzdpmNtxKDaDcDUlckU="+ invalidUserKey = "T5LJHzWwZzdpmN?xKDaDcDUlckU="+ validClientKey = "utD8J485zVIeRCbHyU8yYfw55m2"+ invalidClientKey = "utD8J485zVIeRC?HyU8yYfw55m2"+ keyword = "rev"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRev_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within", secret, clientSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rev,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rev",+ input: fmt.Sprintf("%s token - '%s '\n%s token - '%s '\n", keyword, validUserKey, keyword, validClientKey),+ want: []string{validUserKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within but not valid", inactiveSecret, inactiveClientSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rev,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s '\n%s token - '%s '\n", keyword, invalidUserKey, keyword, invalidClientKey),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Rev.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Rev.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/revampcrm/revampcrm_integration_test.go b/pkg/detectors/revampcrm/revampcrm_integration_test.gonew file mode 100644index 000000000..84ab2dde4--- /dev/null+++ b/pkg/detectors/revampcrm/revampcrm_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package revampcrm++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRevampCRM_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("REVAMPCRM")+ user := testSecrets.MustGetField("ACCOUNT_USER")+ inactiveSecret := testSecrets.MustGetField("REVAMPCRM_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s", secret, user)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RevampCRM,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RevampCRM,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RevampCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RevampCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/revampcrm/revampcrm_test.go b/pkg/detectors/revampcrm/revampcrm_test.goindex 1101321f6..e165ecc91 100644--- a/pkg/detectors/revampcrm/revampcrm_test.go+++ b/pkg/detectors/revampcrm/revampcrm_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package revampcrmimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRevampCRM_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("REVAMPCRM")- user := testSecrets.MustGetField("ACCOUNT_USER")- inactiveSecret := testSecrets.MustGetField("REVAMPCRM_INACTIVE")+var (+ validKey = "JFjlJg7KqFGp5RmsJVq7WDmwObgHW6JEtMOitGuP"+ invalidKey = "JFjlJg7KqFGp5RmsJVq7?DmwObgHW6JEtMOitGuP"+ validId = "JzNRi@oQyzxG>LG7f2oe7R.Xf"+ invalidId = "JzNRi@oQyzxG!LG7f2oe7R.Xf"+ keyword = "revampcrm"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRevampCRM_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s", secret, user)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RevampCRM,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword revampcrm",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RevampCRM,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RevampCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RevampCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/ringcentral/ringcentral_integration_test.go b/pkg/detectors/ringcentral/ringcentral_integration_test.gonew file mode 100644index 000000000..e27386f2c--- /dev/null+++ b/pkg/detectors/ringcentral/ringcentral_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package ringcentral++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRingcentral_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RINGCENTRAL")+ uri := testSecrets.MustGetField("REDIRECTURI")+ inactiveSecret := testSecrets.MustGetField("RINGCENTRAL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s", secret, uri)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RingCentral,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s but not valid", inactiveSecret, uri)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RingCentral,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Ringcentral.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Ringcentral.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/ringcentral/ringcentral_test.go b/pkg/detectors/ringcentral/ringcentral_test.goindex 360028638..5939da8fc 100644--- a/pkg/detectors/ringcentral/ringcentral_test.go+++ b/pkg/detectors/ringcentral/ringcentral_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package ringcentralimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRingcentral_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RINGCENTRAL")- uri := testSecrets.MustGetField("REDIRECTURI")- inactiveSecret := testSecrets.MustGetField("RINGCENTRAL_INACTIVE")+var (+ validKey = "OTXwnia5kOs-SlF3kZEFJK"+ invalidKey = "OTXwnia5kOs?SlF3kZEFJK"+ validUri = "https://wwwSG7$com"+ invalidUri = "https://?wwSG7$com"+ keyword = "ringcentral"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRingcentral_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s", secret, uri)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RingCentral,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword ringcentral",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validUri),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s but not valid", inactiveSecret, uri)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RingCentral,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidUri),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Ringcentral.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Ringcentral.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/ritekit/ritekit_integration_test.go b/pkg/detectors/ritekit/ritekit_integration_test.gonew file mode 100644index 000000000..56bfcfd35--- /dev/null+++ b/pkg/detectors/ritekit/ritekit_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package ritekit++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRiteKit_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RITEKIT")+ inactiveSecret := testSecrets.MustGetField("RITEKIT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ritekit secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RiteKit,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ritekit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RiteKit,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RiteKit.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RiteKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/ritekit/ritekit_test.go b/pkg/detectors/ritekit/ritekit_test.goindex 063651feb..438ac5ed9 100644--- a/pkg/detectors/ritekit/ritekit_test.go+++ b/pkg/detectors/ritekit/ritekit_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package ritekitimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRiteKit_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RITEKIT")- inactiveSecret := testSecrets.MustGetField("RITEKIT_INACTIVE")+var (+ validPattern = "6c2428ff83181c3dcf94e97cb0b15b23599a3cb22857"+ invalidPattern = "6c2428ff83181c3dcf94e9?cb0b15b23599a3cb22857"+ keyword = "ritekit"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRiteKit_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ritekit secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RiteKit,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword ritekit",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ritekit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RiteKit,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RiteKit.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RiteKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/roaring/roaring_integration_test.go b/pkg/detectors/roaring/roaring_integration_test.gonew file mode 100644index 000000000..2be07072f--- /dev/null+++ b/pkg/detectors/roaring/roaring_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package roaring++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRoaring_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ client := testSecrets.MustGetField("ROARING_CLIENT")+ secret := testSecrets.MustGetField("ROARING_SECRET")+ inactiveClient := testSecrets.MustGetField("ROARING_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a roaring secret %s within roaring client %s", secret, client)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Roaring,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a roaring client %s within roaring secret %s but not valid", inactiveClient, secret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Roaring,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Roaring.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Roaring.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/roaring/roaring_test.go b/pkg/detectors/roaring/roaring_test.goindex 640ddb524..fec47e026 100644--- a/pkg/detectors/roaring/roaring_test.go+++ b/pkg/detectors/roaring/roaring_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package roaringimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRoaring_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- client := testSecrets.MustGetField("ROARING_CLIENT")- secret := testSecrets.MustGetField("ROARING_SECRET")- inactiveClient := testSecrets.MustGetField("ROARING_INACTIVE")+var (+ validClient = "-3o_mYkUEBbUKtKjGCg5tgnkzU4T"+ invalidClient = "-3o_mYkUEBbU?tKjGCg5tgnkzU4T"+ validSecret = "XqTQqnAkyH6We_TEK1CRjhwRV51h"+ invalidSecret = "Xq?QqnAkyH6We_TEK1CRjhwRV51h"+ keyword = "roaring"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRoaring_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a roaring secret %s within roaring client %s", secret, client)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Roaring,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword roaring",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validClient, keyword, validSecret),+ want: []string{validSecret + validSecret},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a roaring client %s within roaring secret %s but not valid", inactiveClient, secret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Roaring,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidClient, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Roaring.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Roaring.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rocketreach/rocketreach_integration_test.go b/pkg/detectors/rocketreach/rocketreach_integration_test.gonew file mode 100644index 000000000..651f1c2a8--- /dev/null+++ b/pkg/detectors/rocketreach/rocketreach_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rocketreach++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRocketReach_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ROCKETREACH")+ inactiveSecret := testSecrets.MustGetField("ROCKETREACH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RocketReach,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RocketReach,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RocketReach.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RocketReach.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rocketreach/rocketreach_test.go b/pkg/detectors/rocketreach/rocketreach_test.goindex 651f1c2a8..330977e96 100644--- a/pkg/detectors/rocketreach/rocketreach_test.go+++ b/pkg/detectors/rocketreach/rocketreach_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package rocketreachimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRocketReach_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ROCKETREACH")- inactiveSecret := testSecrets.MustGetField("ROCKETREACH_INACTIVE")+var (+ validPattern = "c897n-ee7bm86g-9vh17ixulnqwz4qmxk2bjf77"+ invalidPattern = "c89?n-ee7bm86g-9vh17ixulnqwz4qmxk2bjf77"+ keyword = "rocketreach"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRocketReach_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RocketReach,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rocketreach",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RocketReach,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RocketReach.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RocketReach.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rockset/rockset_integration_test.go b/pkg/detectors/rockset/rockset_integration_test.gonew file mode 100644index 000000000..e00474186--- /dev/null+++ b/pkg/detectors/rockset/rockset_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rockset++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRockset_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ROCKSET")+ inactiveSecret := testSecrets.MustGetField("ROCKSET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rockset secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rockset,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rockset secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rockset,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Rockset.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Rockset.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rockset/rockset_test.go b/pkg/detectors/rockset/rockset_test.goindex 4d18254b1..36e84755e 100644--- a/pkg/detectors/rockset/rockset_test.go+++ b/pkg/detectors/rockset/rockset_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package rocksetimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRockset_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ROCKSET")- inactiveSecret := testSecrets.MustGetField("ROCKSET_INACTIVE")+var (+ validPattern = "rryYLy3RUXs72dcg6ydXOVWVr6KTeAym0RviPHcIPGafLdzyth0cKelCBa0oJC1q"+ invalidPattern = "rryYLy3RUXs72dcg6ydXOVWVr6KTeAym?RviPHcIPGafLdzyth0cKelCBa0oJC1q"+ keyword = "rockset"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRockset_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rockset secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rockset,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rockset",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rockset secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rockset,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Rockset.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Rockset.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/roninapp/roninapp_integration_test.go b/pkg/detectors/roninapp/roninapp_integration_test.gonew file mode 100644index 000000000..090603947--- /dev/null+++ b/pkg/detectors/roninapp/roninapp_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package roninapp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRoninApp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RONINAPP")+ inactiveSecret := testSecrets.MustGetField("RONINAPP_INACTIVE")+ domain := testSecrets.MustGetField("RONINAPP_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RoninApp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RoninApp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RoninApp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RoninApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/roninapp/roninapp_test.go b/pkg/detectors/roninapp/roninapp_test.goindex 67a47781e..34eff264c 100644--- a/pkg/detectors/roninapp/roninapp_test.go+++ b/pkg/detectors/roninapp/roninapp_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package roninappimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRoninApp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RONINAPP")- inactiveSecret := testSecrets.MustGetField("RONINAPP_INACTIVE")- domain := testSecrets.MustGetField("RONINAPP_DOMAIN")+var (+ validKey = "tmvSxOOxP32WgoPfF2tWEmtsq8"+ invalidKey = "tmvSxOOxP32Wg?PfF2tWEmtsq8"+ validId = "XEZOhulA1oP7vPSc4K"+ invalidId = "?EZOhulA1oP7vPSc4?"+ keyword = "roninapp"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRoninApp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s", secret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RoninApp,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword roninapp",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey, validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RoninApp,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RoninApp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RoninApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/route4me/route4me_integration_test.go b/pkg/detectors/route4me/route4me_integration_test.gonew file mode 100644index 000000000..4f580fd25--- /dev/null+++ b/pkg/detectors/route4me/route4me_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package route4me++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRoute4me_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ROUTE4ME")+ inactiveSecret := testSecrets.MustGetField("ROUTE4ME_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a route4me secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Route4me,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a route4me secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Route4me,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Route4me.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Route4me.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/route4me/route4me_test.go b/pkg/detectors/route4me/route4me_test.goindex de2b1e24e..24e8558fd 100644--- a/pkg/detectors/route4me/route4me_test.go+++ b/pkg/detectors/route4me/route4me_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package route4meimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRoute4me_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ROUTE4ME")- inactiveSecret := testSecrets.MustGetField("ROUTE4ME_INACTIVE")+var (+ validPattern = "WLCO3GIZN6QD2OO9DRNLM4PD7PMH7WH9"+ invalidPattern = "WLCO3GIZN6Q?2OO9DRNLM4PD7PMH7WH9"+ keyword = "route4me"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRoute4me_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a route4me secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Route4me,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword route4me",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a route4me secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Route4me,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Route4me.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Route4me.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/rownd/rownd_integration_test.go b/pkg/detectors/rownd/rownd_integration_test.gonew file mode 100644index 000000000..1fbb340f1--- /dev/null+++ b/pkg/detectors/rownd/rownd_integration_test.go@@ -0,0 +1,119 @@+//go:build detectors+// +build detectors++package rownd++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRownd_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("ROWND_APP_ID")+ key := testSecrets.MustGetField("ROWND")+ secret := testSecrets.MustGetField("ROWND_SECRET")+ inactiveSecret := testSecrets.MustGetField("ROWND_SECRET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s", secret, id, key)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rownd,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s but not valid", inactiveSecret, id, key)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Rownd,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Rownd.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Rownd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/rownd/rownd_test.go b/pkg/detectors/rownd/rownd_test.goindex 1fbb340f1..3fe7b34a7 100644--- a/pkg/detectors/rownd/rownd_test.go+++ b/pkg/detectors/rownd/rownd_test.go@@ -1,118 +1,84 @@-//go:build detectors-// +build detectors-package rowndimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRownd_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("ROWND_APP_ID")- key := testSecrets.MustGetField("ROWND")- secret := testSecrets.MustGetField("ROWND_SECRET")- inactiveSecret := testSecrets.MustGetField("ROWND_SECRET_INACTIVE")+var (+ validKey = "rvhpdb59-6ni1-60q3-jqng-orz5edgkq333"+ invalidKey = "rvhpdb59?6ni1-60q3-jqng-orz5edgkq333"+ validSecret = "uuzywot3x1nlfeogbyft1r9ojb234smw0fnmn7s8kj1s1cc1"+ invalidSecret = "uuzywot3x1nlfeogbyft1r9o?b234smw0fnmn7s8kj1s1cc1"+ validId = "174079606475951818"+ invalidId = "17?079606475951818"+ keyword = "rownd"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRownd_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s", secret, id, key)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rownd,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rownd",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret, keyword, validId),+ want: []string{validKey + validSecret},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s but not valid", inactiveSecret, id, key)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Rownd,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Rownd.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Rownd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/rubygems/rubygems_integration_test.go b/pkg/detectors/rubygems/rubygems_integration_test.gonew file mode 100644index 000000000..bbb51c57f--- /dev/null+++ b/pkg/detectors/rubygems/rubygems_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package rubygems++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRubyGems_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RUBYGEMS_TOKEN")+ inactiveSecret := testSecrets.MustGetField("RUBYGEMS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rubygems secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RubyGems,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a rubygems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RubyGems,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RubyGems.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RubyGems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/rubygems/rubygems_test.go b/pkg/detectors/rubygems/rubygems_test.goindex aa681f83b..eb002e731 100644--- a/pkg/detectors/rubygems/rubygems_test.go+++ b/pkg/detectors/rubygems/rubygems_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package rubygemsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRubyGems_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RUBYGEMS_TOKEN")- inactiveSecret := testSecrets.MustGetField("RUBYGEMS_INACTIVE")+var (+ validPattern = "rubygems_6gbxymmsz3xnk5dg1jc8lm9xekr652c1hr0xt5dp33pn8olh"+ invalidPattern = "rubygems_6gbxymmsz3xnk5dg1jc?lm9xekr652c1hr0xt5dp33pn8olh"+ keyword = "rubygems"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRubyGems_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rubygems secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RubyGems,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword rubygems",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a rubygems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RubyGems,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RubyGems.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RubyGems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/runrunit/runrunit_integration_test.go b/pkg/detectors/runrunit/runrunit_integration_test.gonew file mode 100644index 000000000..a847f8f38--- /dev/null+++ b/pkg/detectors/runrunit/runrunit_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package runrunit++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestRunRunIt_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("RUNRUNIT")+ inactiveSecret := testSecrets.MustGetField("RUNRUNIT_INACTIVE")+ userToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN")+ inactiveUserToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within", secret, userToken)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RunRunIt,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within but not valid", inactiveSecret, inactiveUserToken)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_RunRunIt,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("RunRunIt.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("RunRunIt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/runrunit/runrunit_test.go b/pkg/detectors/runrunit/runrunit_test.goindex 8cdeabd2e..86a42592d 100644--- a/pkg/detectors/runrunit/runrunit_test.go+++ b/pkg/detectors/runrunit/runrunit_test.go@@ -1,122 +1,83 @@-//go:build detectors-// +build detectors-package runrunitimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestRunRunIt_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("RUNRUNIT")- inactiveSecret := testSecrets.MustGetField("RUNRUNIT_INACTIVE")- userToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN")- inactiveUserToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN_INACTIVE")+var (+ validKey = "fdb658d66e09f2caaf9d452e544c3253"+ invalidKey = "fdb658d66e09f2c?af9d452e544c3253"+ validUserToken = "iUm7PtHOhdZxnn4jb5YR"+ invalidUserToken = "iUm7PtHOhd?xnn4jb5YR"+ keyword = "runrunit"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestRunRunIt_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within", secret, userToken)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RunRunIt,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword runrunit",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validUserToken),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within but not valid", inactiveSecret, inactiveUserToken)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_RunRunIt,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidUserToken),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("RunRunIt.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("RunRunIt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/saladcloudapikey/saladcloudapikey_integration_test.go b/pkg/detectors/saladcloudapikey/saladcloudapikey_integration_test.gonew file mode 100644index 000000000..2a3ccda20--- /dev/null+++ b/pkg/detectors/saladcloudapikey/saladcloudapikey_integration_test.go@@ -0,0 +1,161 @@+//go:build detectors+// +build detectors++package saladcloudapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSaladCloudApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SALADCLOUDAPIKEY")+ inactiveSecret := testSecrets.MustGetField("SALADCLOUDAPIKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SaladCloudApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("SaladCloudApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go b/pkg/detectors/saladcloudapikey/saladcloudapikey_test.goindex 95cadb185..8612ca7c8 100644--- a/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go+++ b/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go@@ -1,21 +1,13 @@-//go:build detectors-// +build detectors-package saladcloudapikeyimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")func TestSaladCloudApiKey_Pattern(t *testing.T) {@@ -80,146 +72,3 @@ func TestSaladCloudApiKey_Pattern(t *testing.T) {})}}--func TestSaladCloudApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SALADCLOUDAPIKEY")- inactiveSecret := testSecrets.MustGetField("SALADCLOUDAPIKEY_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SaladCloudApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())- }- }- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("SaladCloudApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }- }- })- }-}diff --git a/pkg/detectors/salesblink/salesblink_integration_test.go b/pkg/detectors/salesblink/salesblink_integration_test.gonew file mode 100644index 000000000..b620ebd9c--- /dev/null+++ b/pkg/detectors/salesblink/salesblink_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package salesblink++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSalesblink_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SALESBLINK_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SALESBLINK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesblink secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesblink,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesblink secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesblink,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Salesblink.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Salesblink.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/salesblink/salesblink_test.go b/pkg/detectors/salesblink/salesblink_test.goindex 43b662ccc..0b2a16161 100644--- a/pkg/detectors/salesblink/salesblink_test.go+++ b/pkg/detectors/salesblink/salesblink_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package salesblinkimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSalesblink_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SALESBLINK_TOKEN")- inactiveSecret := testSecrets.MustGetField("SALESBLINK_INACTIVE")+var (+ validPattern = "key-8DbqWq7r9XKxhUqp3THfA1VMxggreCczfejJsT3UFGISAcEGrrhiA6Ju453dMUXB"+ invalidPattern = "key-8DbqWq7r9XKxhUqp3THfA1VMxggreC?zfejJsT3UFGISAcEGrrhiA6Ju453dMUXB"+ keyword = "salesblink"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSalesblink_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesblink secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesblink,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword salesblink",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesblink secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesblink,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Salesblink.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Salesblink.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/salescookie/salescookie_integration_test.go b/pkg/detectors/salescookie/salescookie_integration_test.gonew file mode 100644index 000000000..66bdccb03--- /dev/null+++ b/pkg/detectors/salescookie/salescookie_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package salescookie++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSalescookie_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SALESCOOKIE")+ inactiveSecret := testSecrets.MustGetField("SALESCOOKIE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salescookie secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salescookie,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salescookie secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salescookie,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Salescookie.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Salescookie.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/salescookie/salescookie_test.go b/pkg/detectors/salescookie/salescookie_test.goindex 48196acf8..70c81a2c2 100644--- a/pkg/detectors/salescookie/salescookie_test.go+++ b/pkg/detectors/salescookie/salescookie_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package salescookieimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSalescookie_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SALESCOOKIE")- inactiveSecret := testSecrets.MustGetField("SALESCOOKIE_INACTIVE")+var (+ validPattern = "4nXYKv]mHDeom5esert30Ea7]4QEfMg6"+ invalidPattern = "4nXYKv]mHD?om5esert30Ea7]4QEfMg6"+ keyword = "salescookie"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSalescookie_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salescookie secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salescookie,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword salescookie",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salescookie secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salescookie,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Salescookie.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Salescookie.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/salesflare/salesflare_integration_test.go b/pkg/detectors/salesflare/salesflare_integration_test.gonew file mode 100644index 000000000..a0a182538--- /dev/null+++ b/pkg/detectors/salesflare/salesflare_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package salesflare++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSalesflare_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SALESFLARE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SALESFLARE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesflare secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesflare,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesflare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesflare,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Salesflare.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Salesflare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/salesflare/salesflare_test.go b/pkg/detectors/salesflare/salesflare_test.goindex e3b0490c7..e70397b90 100644--- a/pkg/detectors/salesflare/salesflare_test.go+++ b/pkg/detectors/salesflare/salesflare_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package salesflareimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSalesflare_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SALESFLARE_TOKEN")- inactiveSecret := testSecrets.MustGetField("SALESFLARE_INACTIVE")+var (+ validPattern = "7dgVfsiOYJhYmw5aQc9DfE6V5jLPuhN2LzsTkhn32KRaZ"+ invalidPattern = "7dgVfsiOYJhYmw5aQc9DfE?V5jLPuhN2LzsTkhn32KRaZ"+ keyword = "salesflare"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSalesflare_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesflare secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesflare,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword salesflare",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesflare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesflare,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Salesflare.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Salesflare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/salesforce/salesforce_integration_test.go b/pkg/detectors/salesforce/salesforce_integration_test.gonew file mode 100644index 000000000..fd5fde6f4--- /dev/null+++ b/pkg/detectors/salesforce/salesforce_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package salesforce++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSalesforce_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ instance := testSecrets.MustGetField("SALESFORCE_INSTANCE")+ // Salesforce secrets are not valid for long, so we may need to regenerate them.+ // Steps to regenerate:+ // 1. Install Salesforce CLI+ // 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault.+ // 3. Run `sfdx org display --targetusername [email protected]`to obtain new token.+ // Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm+ token := testSecrets.MustGetField("SALESFORCE_TOKEN")+ inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesforce,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesforce,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesforce,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesforce,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr)+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }++ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/salesforce/salesforce_test.go b/pkg/detectors/salesforce/salesforce_test.goindex 6c7721e97..99da35666 100644--- a/pkg/detectors/salesforce/salesforce_test.go+++ b/pkg/detectors/salesforce/salesforce_test.go@@ -1,168 +1,82 @@-//go:build detectors-// +build detectors-package salesforceimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSalesforce_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- instance := testSecrets.MustGetField("SALESFORCE_INSTANCE")- // Salesforce secrets are not valid for long, so we may need to regenerate them.- // Steps to regenerate:- // 1. Install Salesforce CLI- // 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault.- // 3. Run `sfdx org display --targetusername [email protected]`to obtain new token.- // Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm- token := testSecrets.MustGetField("SALESFORCE_TOKEN")- inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN")+var (+ validAccessToken = "00MMW9rRmrlVvBi!gZTTPUfk_nkghR_01qsw3NdcHTeDF8dQKWJmiOcfSjfdymrqXH0_vMci6VmxHzrlw07JSfeyMJrA_N89fUJU9vrAYWn_isTE"+ invalidAccessToken = "00MMW9rRmrlVvBi!gZTTPUfk_nkghR_01qsw3NdcHTeDF8dQKWJmiOcf?jfdymrqXH0_vMci6VmxHzrlw07JSfeyMJrA_N89fUJU9vrAYWn_isTE"+ validInstance = "https://wDIMT.HmGz15hePYJBiaiG4leH6y.my.salesforce.com"+ invalidInstance = "https://wDIMT.HmGz15hePYJBi?iG4leH6y.my.salesforce.com"+ keyword = "salesforce"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSalesforce_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesforce,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesforce,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesforce,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - with keyword salesforce",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validAccessToken, keyword, validInstance),+ want: []string{validAccessToken},},{- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesforce,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidAccessToken, keyword, invalidInstance),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}+ return+ }- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/salesmate/salesmate_integration_test.go b/pkg/detectors/salesmate/salesmate_integration_test.gonew file mode 100644index 000000000..c1bc3d341--- /dev/null+++ b/pkg/detectors/salesmate/salesmate_integration_test.go@@ -0,0 +1,118 @@+package salesmate++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSalesmate_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SALESMATE")+ inactiveSecret := testSecrets.MustGetField("SALESMATE_INACTIVE")+ domain := testSecrets.MustGetField("SALESMATE_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesmate secret %s within salesmateDomain %s", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesmate,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a salesmate secret %s within but not valid salesmateDomain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Salesmate,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Salesmate.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Salesmate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/salesmate/salesmate_test.go b/pkg/detectors/salesmate/salesmate_test.goindex d440be9aa..44be1f026 100644--- a/pkg/detectors/salesmate/salesmate_test.go+++ b/pkg/detectors/salesmate/salesmate_test.go@@ -4,115 +4,80 @@ import ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSalesmate_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SALESMATE")- inactiveSecret := testSecrets.MustGetField("SALESMATE_INACTIVE")- domain := testSecrets.MustGetField("SALESMATE_DOMAIN")+var (+ validKey = "398A3699-a4ae-5e7d-ae08-48ef223f1c45"+ invalidKey = "398A3699?a4ae-5e7d-ae08-48ef223f1c45"+ validDomain = "dte2oxDZZSbPhyX1"+ invalidDomain = "?te2oxDZZSbPhyX?"+ keyword = "salesmate"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSalesmate_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesmate secret %s within salesmateDomain %s", secret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesmate,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword salesmate",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),+ want: []string{validKey, validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a salesmate secret %s within but not valid salesmateDomain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Salesmate,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Salesmate.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Salesmate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/saucelabs/saucelabs_integration_test.go b/pkg/detectors/saucelabs/saucelabs_integration_test.gonew file mode 100644index 000000000..7145d0f5d--- /dev/null+++ b/pkg/detectors/saucelabs/saucelabs_integration_test.go@@ -0,0 +1,140 @@+//go:build detectors+// +build detectors++package saucelabs++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSauceLabs_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("SAUCELABS")+ secret := testSecrets.MustGetField("SAUCELABS_SECRET")+ inactiveSecret := testSecrets.MustGetField("SAUCELABS_SECRET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(createFakeString(id, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SauceLabs,+ Verified: true,+ RawV2: []byte(id + secret),+ },+ {+ DetectorType: detectorspb.DetectorType_SauceLabs,+ Verified: false,+ RawV2: []byte(secret + secret),+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("username: %s\n saucelabskey: %s", id, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SauceLabs,+ Verified: false,+ RawV2: []byte(id + inactiveSecret),+ },+ {+ DetectorType: detectorspb.DetectorType_SauceLabs,+ Verified: false,+ RawV2: []byte(inactiveSecret + inactiveSecret),+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SauceLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SauceLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}++func createFakeString(username, key string) string {+ return `+ username: ` + username + `+ saucelabskey: ` + key + `+ `+}diff --git a/pkg/detectors/saucelabs/saucelabs_test.go b/pkg/detectors/saucelabs/saucelabs_test.goindex 7145d0f5d..533feb215 100644--- a/pkg/detectors/saucelabs/saucelabs_test.go+++ b/pkg/detectors/saucelabs/saucelabs_test.go@@ -1,140 +1,85 @@-//go:build detectors-// +build detectors-package saucelabsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSauceLabs_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("SAUCELABS")- secret := testSecrets.MustGetField("SAUCELABS_SECRET")- inactiveSecret := testSecrets.MustGetField("SAUCELABS_SECRET_INACTIVE")+var (+ validUsername = "nBpeh2YF3tjMNmKAUHrDpfXJZJ.ZUhWmObRC735zdh99LeJh4Kcr_2"+ invalidUsername = "?Bpeh?YF3tjMNmKAUHrDpfXJZJ.ZUhWmObRC735zdh99LeJh4Kcr_2"+ validKey = "9toqmh6r-g2ly-k3fr-fiyr-5jnnewlxy3u9"+ invalidKey = "9toqmh6r?g2ly-k3fr-fiyr-5jnnewlxy3u9"+ validBaseUrl = "api.eu-west-0,saucelabs.com"+ invalidBaseUrl = "?pi.eu-west-0,saucelabs.co?"+ keyword = "saucelabs"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSauceLabs_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(createFakeString(id, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SauceLabs,- Verified: true,- RawV2: []byte(id + secret),- },- {- DetectorType: detectorspb.DetectorType_SauceLabs,- Verified: false,- RawV2: []byte(secret + secret),- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("username: %s\n saucelabskey: %s", id, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SauceLabs,- Verified: false,- RawV2: []byte(id + inactiveSecret),- },- {- DetectorType: detectorspb.DetectorType_SauceLabs,- Verified: false,- RawV2: []byte(inactiveSecret + inactiveSecret),- },- },- wantErr: false,+ name: "valid pattern - with keyword saucelabs",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validKey, keyword, validBaseUrl),+ want: []string{".com" + validKey, "token" + validKey},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidKey, keyword, invalidBaseUrl),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SauceLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SauceLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}--func createFakeString(username, key string) string {- return `- username: ` + username + `- saucelabskey: ` + key + `- `-}diff --git a/pkg/detectors/scalewaykey/scalewaykey_integration_test.go b/pkg/detectors/scalewaykey/scalewaykey_integration_test.gonew file mode 100644index 000000000..409b92dcd--- /dev/null+++ b/pkg/detectors/scalewaykey/scalewaykey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scalewaykey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScalewayKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCALEWAYKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SCALEWAYKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScalewayKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScalewayKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScalewayKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScalewayKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scalewaykey/scalewaykey_test.go b/pkg/detectors/scalewaykey/scalewaykey_test.goindex 58bbd79a0..5b4f37481 100644--- a/pkg/detectors/scalewaykey/scalewaykey_test.go+++ b/pkg/detectors/scalewaykey/scalewaykey_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scalewaykeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScalewayKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCALEWAYKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("SCALEWAYKEY_INACTIVE")+var (+ validPattern = "lj6z1t7m-hdu4-647t-uoju-nlfnizsv7zry"+ invalidPattern = "lj6z1t7m?hdu4-647t-uoju-nlfnizsv7zry"+ keyword = "scalewaykey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScalewayKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScalewayKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scalewaykey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScalewayKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScalewayKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScalewayKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scalr/scalr_integration_test.go b/pkg/detectors/scalr/scalr_integration_test.gonew file mode 100644index 000000000..e1103d10f--- /dev/null+++ b/pkg/detectors/scalr/scalr_integration_test.go@@ -0,0 +1,126 @@+package scalr++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScalr_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCALR")+ inactiveSecret := testSecrets.MustGetField("SCALR_INACTIVE")+ id := testSecrets.MustGetField("SCALR_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scalr secret %s within scalr %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Scalr,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Scalr,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scalr secret %s within but not valid scalr %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Scalr,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_Scalr,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Scalr.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Scalr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scalr/scalr_test.go b/pkg/detectors/scalr/scalr_test.goindex 2fd07bf70..cba85fa0b 100644--- a/pkg/detectors/scalr/scalr_test.go+++ b/pkg/detectors/scalr/scalr_test.go@@ -4,123 +4,80 @@ import ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScalr_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCALR")- inactiveSecret := testSecrets.MustGetField("SCALR_INACTIVE")- id := testSecrets.MustGetField("SCALR_DOMAIN")+var (+ validKey = "bmy2Q33nZnlwZ5wCu_umGbBHF_6547oPzvAGLpUEwhWwOJmU5DUlZIOWZ0Yr7g2Yt4s3ZyyfWxzOlLLOmkGXpzuQ3.gEhgGUHUngyyiPIZjVoYQ.W0O8k4G3JBC705AXkXxANwrF"+ invalidKey = "bmy2Q33nZnlwZ5wCu?umGbBHF?6547oPzvAGLpUEwhWwOJmU5DUlZIOWZ0Yr7g2Yt4s3ZyyfWxzOlLLOmkGXpzuQ3?gEhgGUHUngyyiPIZjVoYQ.W0O8k4G3JBC705AXkXxANwrF"+ validId = "h7bcggvlzhq6t5m47f37jav"+ invalidId = "?7bcgg?lzhq6t5m47f37jav"+ keyword = "scalr"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScalr_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scalr secret %s within scalr %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Scalr,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Scalr,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scalr",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey, validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scalr secret %s within but not valid scalr %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Scalr,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_Scalr,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Scalr.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Scalr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scrapeowl/scrapeowl_integration_test.go b/pkg/detectors/scrapeowl/scrapeowl_integration_test.gonew file mode 100644index 000000000..3b38277fc--- /dev/null+++ b/pkg/detectors/scrapeowl/scrapeowl_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scrapeowl++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScrapeowl_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPEOWL")+ inactiveSecret := testSecrets.MustGetField("SCRAPEOWL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Scrapeowl,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Scrapeowl,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Scrapeowl.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Scrapeowl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scrapeowl/scrapeowl_test.go b/pkg/detectors/scrapeowl/scrapeowl_test.goindex 519f9a03c..1d05ef18f 100644--- a/pkg/detectors/scrapeowl/scrapeowl_test.go+++ b/pkg/detectors/scrapeowl/scrapeowl_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scrapeowlimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScrapeowl_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPEOWL")- inactiveSecret := testSecrets.MustGetField("SCRAPEOWL_INACTIVE")+var (+ validPattern = "dwkiwm366z8gqutb1ukv9sztut04be"+ invalidPattern = "dwkiwm366z8gqut?1ukv9sztut04be"+ keyword = "scrapeowl"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScrapeowl_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Scrapeowl,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scrapeowl",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Scrapeowl,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Scrapeowl.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Scrapeowl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scraperapi/scraperapi_integration_test.go b/pkg/detectors/scraperapi/scraperapi_integration_test.gonew file mode 100644index 000000000..8b8cec04d--- /dev/null+++ b/pkg/detectors/scraperapi/scraperapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scraperapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScraperAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPERAPI")+ inactiveSecret := testSecrets.MustGetField("SCRAPERAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScraperAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScraperAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScraperAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScraperAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scraperapi/scraperapi_test.go b/pkg/detectors/scraperapi/scraperapi_test.goindex c809f33d8..100c82aca 100644--- a/pkg/detectors/scraperapi/scraperapi_test.go+++ b/pkg/detectors/scraperapi/scraperapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scraperapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScraperAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPERAPI")- inactiveSecret := testSecrets.MustGetField("SCRAPERAPI_INACTIVE")+var (+ validPattern = "28ksnqsrn7q12wp11onrmr4bh1dvi7ut"+ invalidPattern = "28ksnqsrn7q?2wp11onrmr4bh1dvi7ut"+ keyword = "scraperapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScraperAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScraperAPI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scraperapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScraperAPI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScraperAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScraperAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scraperbox/scraperbox_integration_test.go b/pkg/detectors/scraperbox/scraperbox_integration_test.gonew file mode 100644index 000000000..22234582c--- /dev/null+++ b/pkg/detectors/scraperbox/scraperbox_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scraperbox++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScraperBox_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPERBOX")+ inactiveSecret := testSecrets.MustGetField("SCRAPERBOX_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScraperBox,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScraperBox,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScraperBox.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScraperBox.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scraperbox/scraperbox_test.go b/pkg/detectors/scraperbox/scraperbox_test.goindex 4a867efbf..ef7c06dfd 100644--- a/pkg/detectors/scraperbox/scraperbox_test.go+++ b/pkg/detectors/scraperbox/scraperbox_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scraperboximport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScraperBox_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPERBOX")- inactiveSecret := testSecrets.MustGetField("SCRAPERBOX_INACTIVE")+var (+ validPattern = "H8AOZKMTK849QUN2CAKZ8L77YS53I34L"+ invalidPattern = "H8AOZKMTK849QUN2?AKZ8L77YS53I34L"+ keyword = "scraperbox"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScraperBox_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScraperBox,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scraperbox",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScraperBox,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScraperBox.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScraperBox.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scrapestack/scrapestack_integration_test.go b/pkg/detectors/scrapestack/scrapestack_integration_test.gonew file mode 100644index 000000000..5560c56f8--- /dev/null+++ b/pkg/detectors/scrapestack/scrapestack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scrapestack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScrapeStack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPESTACK")+ inactiveSecret := testSecrets.MustGetField("SCRAPESTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrapeStack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrapeStack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScrapeStack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScrapeStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scrapestack/scrapestack_test.go b/pkg/detectors/scrapestack/scrapestack_test.goindex 0fdb02558..a46fa6c9c 100644--- a/pkg/detectors/scrapestack/scrapestack_test.go+++ b/pkg/detectors/scrapestack/scrapestack_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scrapestackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScrapeStack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPESTACK")- inactiveSecret := testSecrets.MustGetField("SCRAPESTACK_INACTIVE")+var (+ validPattern = "gdozp26v2hzq91fh8k412n5ik5klj59p"+ invalidPattern = "gdozp26v2hzq91fh?k412n5ik5klj59p"+ keyword = "scrapestack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScrapeStack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrapeStack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scrapestack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrapeStack,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScrapeStack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScrapeStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scrapfly/scrapfly_integration_test.go b/pkg/detectors/scrapfly/scrapfly_integration_test.gonew file mode 100644index 000000000..1634b6583--- /dev/null+++ b/pkg/detectors/scrapfly/scrapfly_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scrapfly++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScrapfly_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPFLY")+ inactiveSecret := testSecrets.MustGetField("SCRAPFLY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Scrapfly,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Scrapfly,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Scrapfly.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Scrapfly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scrapfly/scrapfly_test.go b/pkg/detectors/scrapfly/scrapfly_test.goindex 34a7ff5b0..71a844edc 100644--- a/pkg/detectors/scrapfly/scrapfly_test.go+++ b/pkg/detectors/scrapfly/scrapfly_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scrapflyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScrapfly_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPFLY")- inactiveSecret := testSecrets.MustGetField("SCRAPFLY_INACTIVE")+var (+ validPattern = "ajw59xcg6r6uywscubwxh5t3qmji2tp2"+ invalidPattern = "ajw59xcg6r6?ywscubwxh5t3qmji2tp2"+ keyword = "scrapfly"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScrapfly_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Scrapfly,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scrapfly",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Scrapfly,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Scrapfly.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Scrapfly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scrapingant/scrapingant_integration_test.go b/pkg/detectors/scrapingant/scrapingant_integration_test.gonew file mode 100644index 000000000..eb69636ca--- /dev/null+++ b/pkg/detectors/scrapingant/scrapingant_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scrapingant++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScrapingAnt_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPINGANT")+ inactiveSecret := testSecrets.MustGetField("SCRAPINGANT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrapingAnt,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrapingAnt,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScrapingAnt.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScrapingAnt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scrapingant/scrapingant_test.go b/pkg/detectors/scrapingant/scrapingant_test.goindex eb69636ca..dcce62fe9 100644--- a/pkg/detectors/scrapingant/scrapingant_test.go+++ b/pkg/detectors/scrapingant/scrapingant_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scrapingantimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScrapingAnt_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPINGANT")- inactiveSecret := testSecrets.MustGetField("SCRAPINGANT_INACTIVE")+var (+ validPattern = "uifssyi7y40rf7ryyr3vczj8xbhbv49f"+ invalidPattern = "uifss?i7y40rf7ryyr3vczj8xbhbv49f"+ keyword = "scrapingant"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScrapingAnt_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrapingAnt,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scrapingant",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrapingAnt,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScrapingAnt.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScrapingAnt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scrapingbee/scrapingbee_integration_test.go b/pkg/detectors/scrapingbee/scrapingbee_integration_test.gonew file mode 100644index 000000000..2fb482ed2--- /dev/null+++ b/pkg/detectors/scrapingbee/scrapingbee_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scrapingbee++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScrapingBee_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRAPINGBEE")+ inactiveSecret := testSecrets.MustGetField("SCRAPINGBEE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrapingBee,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrapingBee,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScrapingBee.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScrapingBee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scrapingbee/scrapingbee_test.go b/pkg/detectors/scrapingbee/scrapingbee_test.goindex 561c67eb2..24061f0a4 100644--- a/pkg/detectors/scrapingbee/scrapingbee_test.go+++ b/pkg/detectors/scrapingbee/scrapingbee_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scrapingbeeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScrapingBee_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPINGBEE")- inactiveSecret := testSecrets.MustGetField("SCRAPINGBEE_INACTIVE")+var (+ validPattern = "HOLTCTPVL3V35NK795FY6INWQKTTTSGL6W26JKHLC3CGUNJ1DV35SWPUAZ26BUX2EEZZ7T2SJBOL8WZB"+ invalidPattern = "HOLT?TPVL3V35NK795FY6INWQKTTTSGL6W26JKHLC3CGUNJ1DV35SWPUAZ26BUX2EEZZ7T2SJBOL8WZB"+ keyword = "scrapingbee"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScrapingBee_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrapingBee,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scrapingbee",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrapingBee,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScrapingBee.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScrapingBee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/screenshotapi/screenshotapi_integration_test.go b/pkg/detectors/screenshotapi/screenshotapi_integration_test.gonew file mode 100644index 000000000..61cfe00e9--- /dev/null+++ b/pkg/detectors/screenshotapi/screenshotapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package screenshotapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScreenshotAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCREENSHOTAPI")+ inactiveSecret := testSecrets.MustGetField("SCREENSHOTAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScreenshotAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScreenshotAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/screenshotapi/screenshotapi_test.go b/pkg/detectors/screenshotapi/screenshotapi_test.goindex 34601e49b..2c028fdd5 100644--- a/pkg/detectors/screenshotapi/screenshotapi_test.go+++ b/pkg/detectors/screenshotapi/screenshotapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package screenshotapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScreenshotAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCREENSHOTAPI")- inactiveSecret := testSecrets.MustGetField("SCREENSHOTAPI_INACTIVE")+var (+ validPattern = "5QCXUYE-B9LNS58-X62XLXO-CHZJZDO"+ invalidPattern = "5QCXUYE?B9LNS58-X62XLXO-CHZJZDO"+ keyword = "screenshotapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScreenshotAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScreenshotAPI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword screenshotapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScreenshotAPI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/screenshotlayer/screenshotlayer_integration_test.go b/pkg/detectors/screenshotlayer/screenshotlayer_integration_test.gonew file mode 100644index 000000000..53a431204--- /dev/null+++ b/pkg/detectors/screenshotlayer/screenshotlayer_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package screenshotlayer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScreenshotLayer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCREENSHOTLAYER")+ inactiveSecret := testSecrets.MustGetField("SCREENSHOTLAYER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScreenshotLayer,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScreenshotLayer,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScreenshotLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScreenshotLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/screenshotlayer/screenshotlayer_test.go b/pkg/detectors/screenshotlayer/screenshotlayer_test.goindex 15f03a2d7..7f7538bbc 100644--- a/pkg/detectors/screenshotlayer/screenshotlayer_test.go+++ b/pkg/detectors/screenshotlayer/screenshotlayer_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package screenshotlayerimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScreenshotLayer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCREENSHOTLAYER")- inactiveSecret := testSecrets.MustGetField("SCREENSHOTLAYER_INACTIVE")+var (+ validPattern = "mbACjpi8P_c_6y2WMu4dUfvnvKWuDb_3"+ invalidPattern = "mbACjpi8P_c_6y2W?u4dUfvnvKWuDb_3"+ keyword = "screenshotlayer"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScreenshotLayer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScreenshotLayer,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword screenshotlayer",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScreenshotLayer,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScreenshotLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScreenshotLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/scrutinizerci/scrutinizerci_integration_test.go b/pkg/detectors/scrutinizerci/scrutinizerci_integration_test.gonew file mode 100644index 000000000..eaf8439f9--- /dev/null+++ b/pkg/detectors/scrutinizerci/scrutinizerci_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package scrutinizerci++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestScrutinizerCi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SCRUTINIZERCI")+ inactiveSecret := testSecrets.MustGetField("SCRUTINIZERCI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrutinizerCi,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ScrutinizerCi,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ScrutinizerCi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ScrutinizerCi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/scrutinizerci/scrutinizerci_test.go b/pkg/detectors/scrutinizerci/scrutinizerci_test.goindex f43c24e8e..e4798ea40 100644--- a/pkg/detectors/scrutinizerci/scrutinizerci_test.go+++ b/pkg/detectors/scrutinizerci/scrutinizerci_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package scrutinizerciimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestScrutinizerCi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRUTINIZERCI")- inactiveSecret := testSecrets.MustGetField("SCRUTINIZERCI_INACTIVE")+var (+ validPattern = "tj8dzw8on7hw5j5wtiphzc4hv5wfvb2kbk4djyz8r1oxqes8otd3ms0mrqw8jlwt"+ invalidPattern = "tj8dzw8on7hw5j5wtiphzc4hv5wfv?2kbk4djyz8r1oxqes8otd3ms0mrqw8jlwt"+ keyword = "scrutinizerci"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestScrutinizerCi_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrutinizerCi,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword scrutinizerci",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScrutinizerCi,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScrutinizerCi.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScrutinizerCi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/securitytrails/securitytrails_integration_test.go b/pkg/detectors/securitytrails/securitytrails_integration_test.gonew file mode 100644index 000000000..73f9bccc5--- /dev/null+++ b/pkg/detectors/securitytrails/securitytrails_integration_test.go@@ -0,0 +1,136 @@+//go:build detectors+// +build detectors++package securitytrails++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSecurityTrails_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SECURITYTRAILS")+ inactiveSecret := testSecrets.MustGetField("SECURITYTRAILS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a securitytrails secret\n %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SecurityTrails,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified inline",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SecurityTrails,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SecurityTrails,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SecurityTrails.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SecurityTrails.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/securitytrails/securitytrails_test.go b/pkg/detectors/securitytrails/securitytrails_test.goindex 952023720..83dd8f0a1 100644--- a/pkg/detectors/securitytrails/securitytrails_test.go+++ b/pkg/detectors/securitytrails/securitytrails_test.go@@ -1,136 +1,91 @@-//go:build detectors-// +build detectors-package securitytrailsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSecurityTrails_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SECURITYTRAILS")- inactiveSecret := testSecrets.MustGetField("SECURITYTRAILS_INACTIVE")+var (+ validPattern = "wZ8KxGUFd2G1IrazudQo9UpGyqr8jji6"+ invalidPattern = "wZ8KxGUFd2G1Iraz?dQo9UpGyqr8jji6"+ keyword = "securitytrails"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSecurityTrails_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a securitytrails secret\n %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SecurityTrails,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword securitytrails",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, verified inline",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SecurityTrails,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SecurityTrails,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SecurityTrails.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SecurityTrails.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/segmentapikey/segmentapikey_integration_test.go b/pkg/detectors/segmentapikey/segmentapikey_integration_test.gonew file mode 100644index 000000000..9f34cb1bb--- /dev/null+++ b/pkg/detectors/segmentapikey/segmentapikey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package segmentapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSegmentApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SEGMENTAPIKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SEGMENTAPIKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SegmentApiKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SegmentApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SegmentApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SegmentApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/segmentapikey/segmentapikey_test.go b/pkg/detectors/segmentapikey/segmentapikey_test.goindex 133f92853..9344a12b2 100644--- a/pkg/detectors/segmentapikey/segmentapikey_test.go+++ b/pkg/detectors/segmentapikey/segmentapikey_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package segmentapikeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSegmentApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SEGMENTAPIKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("SEGMENTAPIKEY_INACTIVE")+var (+ validPattern = "ENTvbxIazIP8g_chwXUfz4M930vr4YArS00iBPtBADI.E7RjExdT9CXgjFO-EhFE1WQGjRflI9YSJG-cCYPUB5x"+ invalidPattern = "ENTvbxIazIP8g_chwXUfz4M930vr4YArS00iBPtBADI?E7RjExdT9CXgjFO-EhFE1WQGjRflI9YSJG-cCYPUB5x"+ keyword = "segmentapikey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSegmentApiKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SegmentApiKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword segmentapikey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SegmentApiKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SegmentApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SegmentApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/selectpdf/selectpdf_integration_test.go b/pkg/detectors/selectpdf/selectpdf_integration_test.gonew file mode 100644index 000000000..30906236d--- /dev/null+++ b/pkg/detectors/selectpdf/selectpdf_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package selectpdf++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSelectPDF_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SELECTPDF")+ inactiveSecret := testSecrets.MustGetField("SELECTPDF_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SelectPDF,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SelectPDF,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SelectPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SelectPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/selectpdf/selectpdf_test.go b/pkg/detectors/selectpdf/selectpdf_test.goindex 6757a7f1f..9da69e3dd 100644--- a/pkg/detectors/selectpdf/selectpdf_test.go+++ b/pkg/detectors/selectpdf/selectpdf_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package selectpdfimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSelectPDF_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SELECTPDF")- inactiveSecret := testSecrets.MustGetField("SELECTPDF_INACTIVE")+var (+ validPattern = "fmajqnwjj-ia1l4in7-4r6ktkn1ikiy89x02"+ invalidPattern = "fmajqnwjj-ia1l4in7?4r6ktkn1ikiy89x02"+ keyword = "selectpdf"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSelectPDF_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SelectPDF,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword selectpdf",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SelectPDF,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SelectPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SelectPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/semaphore/semaphore_integration_test.go b/pkg/detectors/semaphore/semaphore_integration_test.gonew file mode 100644index 000000000..c107c1ae9--- /dev/null+++ b/pkg/detectors/semaphore/semaphore_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package semaphore++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSemaphore_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SEMAPHORE")+ inactiveSecret := testSecrets.MustGetField("SEMAPHORE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a semaphore secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Semaphore,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a semaphore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Semaphore,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Semaphore.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Semaphore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/semaphore/semaphore_test.go b/pkg/detectors/semaphore/semaphore_test.goindex 116535e06..60b73d46e 100644--- a/pkg/detectors/semaphore/semaphore_test.go+++ b/pkg/detectors/semaphore/semaphore_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package semaphoreimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSemaphore_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SEMAPHORE")- inactiveSecret := testSecrets.MustGetField("SEMAPHORE_INACTIVE")+var (+ validPattern = "e90jcvaioqwdzi0thnnpx59nv2k4efom"+ invalidPattern = "e90jcvaioqwdzi0t?nnpx59nv2k4efom"+ keyword = "semaphore.co"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSemaphore_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a semaphore secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Semaphore,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword semaphore.co",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a semaphore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Semaphore,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Semaphore.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Semaphore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sendbird/sendbird_integration_test.go b/pkg/detectors/sendbird/sendbird_integration_test.gonew file mode 100644index 000000000..aae61bfb8--- /dev/null+++ b/pkg/detectors/sendbird/sendbird_integration_test.go@@ -0,0 +1,193 @@+//go:build detectors+// +build detectors++package sendbird++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSendbird_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SENDBIRD")+ inactiveSecret := testSecrets.MustGetField("SENDBIRD_INACTIVE")+ appId := testSecrets.MustGetField("SENDBIRD_APP_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sendbird,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within but not valid", appId, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sendbird,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sendbird,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sendbird,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected error code",+ s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sendbird,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but failed to decode json",+ s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sendbird,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/sendbird/sendbird_test.go b/pkg/detectors/sendbird/sendbird_test.goindex b84849869..87ae80568 100644--- a/pkg/detectors/sendbird/sendbird_test.go+++ b/pkg/detectors/sendbird/sendbird_test.go@@ -1,191 +1,82 @@-//go:build detectors-// +build detectors-package sendbirdimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSendbird_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SENDBIRD")- inactiveSecret := testSecrets.MustGetField("SENDBIRD_INACTIVE")- appId := testSecrets.MustGetField("SENDBIRD_APP_ID")+var (+ validKey = "e9aeb40d34e6a7a9fb10e5876be968e344420b0c"+ invalidKey = "?9aeb40d34e6a7a9fb10e5876be968e344420b0c"+ validAppId = "FAB7BE5B-E6DD-670A-A3DC-91AF003DB677"+ invalidAppId = "FAB7BE5B?E6DD-670A-A3DC-91AF003DB677"+ keyword = "sendbird"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSendbird_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sendbird,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within but not valid", appId, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sendbird,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sendbird,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sendbird,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - with keyword sendbird",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validAppId),+ want: []string{validKey},},{- name: "found, verified but unexpected error code",- s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sendbird,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but failed to decode json",- s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sendbird,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidAppId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_integration_test.go b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_integration_test.gonew file mode 100644index 000000000..7d1b856ad--- /dev/null+++ b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_integration_test.go@@ -0,0 +1,158 @@+//go:build detectors+// +build detectors++package sendbirdorganizationapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI")+ inactiveSecret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }++ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.goindex 7d1b856ad..aa3d17e16 100644--- a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go+++ b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go@@ -1,157 +1,90 @@-//go:build detectors-// +build detectors-package sendbirdorganizationapiimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI")- inactiveSecret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI_INACTIVE")+var (+ validPattern = "fd3ddab708ffd4b71f079272"+ invalidPattern = "f?3ddab708ffd4b71f079272"+ keyword = "sendbirdorganizationapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSendbirdOrganizationAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword sendbirdorganizationapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/sendgrid/sendgrid_integration_test.go b/pkg/detectors/sendgrid/sendgrid_integration_test.gonew file mode 100644index 000000000..230cf1825--- /dev/null+++ b/pkg/detectors/sendgrid/sendgrid_integration_test.go@@ -0,0 +1,152 @@+//go:build detectors+// +build detectors++package sendgrid++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSendgrid_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SENDGRID")+ secretInactive := testSecrets.MustGetField("SENDGRID_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendGrid,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendGrid,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendGrid,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendGrid,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo", "ExtraData")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sendgrid/sendgrid_test.go b/pkg/detectors/sendgrid/sendgrid_test.goindex 886f5ffb0..254a33ca2 100644--- a/pkg/detectors/sendgrid/sendgrid_test.go+++ b/pkg/detectors/sendgrid/sendgrid_test.go@@ -1,150 +1,85 @@-//go:build detectors-// +build detectors-package sendgridimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSendgrid_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SENDGRID")- secretInactive := testSecrets.MustGetField("SENDGRID_INACTIVE")- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validPattern = "SG.ZV-VYwtHzJJW4wF8yPQk3.ZXg9c9DZuOgUcW1f2inP6SqfEsYG82zAe0wG7brZZ5OvruV-I"+ invalidPattern = "SG.ZV-VYwtHzJJW4wF8yPQk3.ZXg9?9DZuOgUcW1f2inP6SqfEsYG82zAe0wG7brZZ5OvruV-I"+ keyword = "sendgrid"+)++func TestSendgrid_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendGrid,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword sendgrid",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendGrid,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendGrid,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendGrid,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo", "ExtraData")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/sendinbluev2/sendinbluev2_integration_test.go b/pkg/detectors/sendinbluev2/sendinbluev2_integration_test.gonew file mode 100644index 000000000..0e3546a77--- /dev/null+++ b/pkg/detectors/sendinbluev2/sendinbluev2_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package sendinbluev2++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSendinBlueV2_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SENDINBLUEV2_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SENDINBLUEV2_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendinBlueV2,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within but unverified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SendinBlueV2,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SendinBlueV2.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SendinBlueV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sendinbluev2/sendinbluev2_test.go b/pkg/detectors/sendinbluev2/sendinbluev2_test.goindex 0e3546a77..5f3886111 100644--- a/pkg/detectors/sendinbluev2/sendinbluev2_test.go+++ b/pkg/detectors/sendinbluev2/sendinbluev2_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package sendinbluev2import ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSendinBlueV2_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SENDINBLUEV2_TOKEN")- inactiveSecret := testSecrets.MustGetField("SENDINBLUEV2_INACTIVE")+var (+ validPattern = "xkeysib-C6S1LXk_u4mw_uIss4MGmJpH8yrOwFep2aN5YLALYVpAb4buJ7uvxqYfrb3kZL5ao2JvUEFb1vRk79IXj"+ invalidPattern = "xkeysib-C6S1LXk_u4mw_uIss4MGmJpH8yrOwFep2aN5?LALYVpAb4buJ7uvxqYfrb3kZL5ao2JvUEFb1vRk79IXj"+ keyword = "sendinbluev2"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSendinBlueV2_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendinBlueV2,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within but unverified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SendinBlueV2,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword sendinbluev2",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SendinBlueV2.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SendinBlueV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sentrytoken/sentrytoken_integration_test.go b/pkg/detectors/sentrytoken/sentrytoken_integration_test.gonew file mode 100644index 000000000..34729e950--- /dev/null+++ b/pkg/detectors/sentrytoken/sentrytoken_integration_test.go@@ -0,0 +1,229 @@+//go:build detectors+// +build detectors++package sentrytoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSentryToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SENTRYTOKEN_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SENTRYTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, would be verified but for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found and valid but unexpected api response",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, good key but wrong scope",+ s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, account deactivated",+ s: Scanner{client: common.ConstantResponseHttpClient(200, responseAccountDeactivated)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, account deactivated",+ s: Scanner{client: common.ConstantResponseHttpClient(200, responseEmpty)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SentryToken,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, opts); diff != "" {+ t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++const (+ responseBody403 = `+[+ {+ "organization": {+ "id": "911964",+ "slug": "wigslap",+ "status": {+ "id": "active",+ "name": "active"+ },+ "name": "wigslap"+ }+ }+]+`+ responseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`+ responseEmpty = `[]`+)++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sentrytoken/sentrytoken_test.go b/pkg/detectors/sentrytoken/sentrytoken_test.goindex 5c64febff..8c7806ff4 100644--- a/pkg/detectors/sentrytoken/sentrytoken_test.go+++ b/pkg/detectors/sentrytoken/sentrytoken_test.go@@ -1,229 +1,90 @@-//go:build detectors-// +build detectors-package sentrytokenimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSentryToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SENTRYTOKEN_TOKEN")- inactiveSecret := testSecrets.MustGetField("SENTRYTOKEN_INACTIVE")+var (+ validPattern = "ad00eba0e2b5b057146e1b2b9373f86dbb0e712d106529111d97cb13f081de20"+ invalidPattern = "ad00e?a0e2b5b057146e1b2b9373f86dbb0e712d106529111d97cb13f081de20"+ keyword = "sentrytoken"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSentryToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "found, would be verified but for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found and valid but unexpected api response",- s: Scanner{client: common.ConstantResponseHttpClient(500, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, good key but wrong scope",- s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sentrytoken",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, account deactivated",- s: Scanner{client: common.ConstantResponseHttpClient(200, responseAccountDeactivated)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, account deactivated",- s: Scanner{client: common.ConstantResponseHttpClient(200, responseEmpty)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SentryToken,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, opts); diff != "" {- t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-const (- responseBody403 = `-[- {- "organization": {- "id": "911964",- "slug": "wigslap",- "status": {- "id": "active",- "name": "active"- },- "name": "wigslap"- }- }-]-`- responseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`- responseEmpty = `[]`-)--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/serphouse/serphouse_integration_test.go b/pkg/detectors/serphouse/serphouse_integration_test.gonew file mode 100644index 000000000..ed6fa8c68--- /dev/null+++ b/pkg/detectors/serphouse/serphouse_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package serphouse++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSerphouse_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SERPHOUSE")+ inactiveSecret := testSecrets.MustGetField("SERPHOUSE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a serphouse secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Serphouse,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a serphouse secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Serphouse,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Serphouse.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Serphouse.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/serphouse/serphouse_test.go b/pkg/detectors/serphouse/serphouse_test.goindex 9182a9e4c..48e1cfd41 100644--- a/pkg/detectors/serphouse/serphouse_test.go+++ b/pkg/detectors/serphouse/serphouse_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package serphouseimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSerphouse_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SERPHOUSE")- inactiveSecret := testSecrets.MustGetField("SERPHOUSE_INACTIVE")+var (+ validPattern = "H8VuKAcg3pmwH3NbhOSDJSbVqCxslaOQ7lVqF61be25HEGzCJXzzCGoxw8sE"+ invalidPattern = "H8VuKAcg3pmwH3Nbh?SDJSbVqCxslaOQ7lVqF61be25HEGzCJXzzCGoxw8sE"+ keyword = "serphouse"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSerphouse_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a serphouse secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Serphouse,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword serphouse",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a serphouse secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Serphouse,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Serphouse.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Serphouse.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/serpstack/serpstack_integration_test.go b/pkg/detectors/serpstack/serpstack_integration_test.gonew file mode 100644index 000000000..84cca277c--- /dev/null+++ b/pkg/detectors/serpstack/serpstack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package serpstack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSerpStack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SERPSTACK")+ inactiveSecret := testSecrets.MustGetField("SERPSTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a serpstack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SerpStack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a serpstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SerpStack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SerpStack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SerpStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/serpstack/serpstack_test.go b/pkg/detectors/serpstack/serpstack_test.goindex 84cca277c..d2b07e7cb 100644--- a/pkg/detectors/serpstack/serpstack_test.go+++ b/pkg/detectors/serpstack/serpstack_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package serpstackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSerpStack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SERPSTACK")- inactiveSecret := testSecrets.MustGetField("SERPSTACK_INACTIVE")+var (+ validPattern = "jeqozza8xili4i96pc1x1ww67lmvrw27"+ invalidPattern = "jeqozza8xili4i96?c1x1ww67lmvrw27"+ keyword = "serpstack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSerpStack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a serpstack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SerpStack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword serpstack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a serpstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SerpStack,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SerpStack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SerpStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sheety/sheety_integration_test.go b/pkg/detectors/sheety/sheety_integration_test.gonew file mode 100644index 000000000..3c68d97ed--- /dev/null+++ b/pkg/detectors/sheety/sheety_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package sheety++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSheety_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHEETY")+ id := testSecrets.MustGetField("SHEETY_ID")++ inactiveSecret := testSecrets.MustGetField("SHEETY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sheety,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sheety,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sheety.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sheety.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sheety/sheety_test.go b/pkg/detectors/sheety/sheety_test.goindex d79b45a5f..de4b45f5a 100644--- a/pkg/detectors/sheety/sheety_test.go+++ b/pkg/detectors/sheety/sheety_test.go@@ -1,122 +1,83 @@-//go:build detectors-// +build detectors-package sheetyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSheety_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHEETY")- id := testSecrets.MustGetField("SHEETY_ID")-- inactiveSecret := testSecrets.MustGetField("SHEETY_INACTIVE")+var (+ validKey = "qtb537zc8slokboyekq08tsut9atqa4hnewgjsi2olklupy2z4s3wilqorymakjd"+ invalidKey = "qtb537zc8slokboyekq08tsut9atqa4h?ewgjsi2olklupy2z4s3wilqorymakjd"+ validId = "81no0tu89hxci6swi979krdkv87f21z9"+ invalidId = "81no0tu89hxc?6swi979krdkv87f21z9"+ keyword = "sheety"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSheety_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sheety,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sheety,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword sheety",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sheety.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Sheety.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sherpadesk/sherpadesk_integration_test.go b/pkg/detectors/sherpadesk/sherpadesk_integration_test.gonew file mode 100644index 000000000..14145cc98--- /dev/null+++ b/pkg/detectors/sherpadesk/sherpadesk_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package sherpadesk++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSherpadesk_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHERPADESK")+ inactiveSecret := testSecrets.MustGetField("SHERPADESK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sherpadesk,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sherpadesk,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sherpadesk.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sherpadesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sherpadesk/sherpadesk_test.go b/pkg/detectors/sherpadesk/sherpadesk_test.goindex c99dd325d..a9f63c4e0 100644--- a/pkg/detectors/sherpadesk/sherpadesk_test.go+++ b/pkg/detectors/sherpadesk/sherpadesk_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package sherpadeskimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSherpadesk_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHERPADESK")- inactiveSecret := testSecrets.MustGetField("SHERPADESK_INACTIVE")+var (+ validPattern = "g8wjz4xjtdaoswrmbl2ypcu0neq4cvr4"+ invalidPattern = "g8wjz4xjtdaoswrm?l2ypcu0neq4cvr4"+ keyword = "sherpadesk"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSherpadesk_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sherpadesk,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sherpadesk",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sherpadesk,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sherpadesk.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Sherpadesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shipday/shipday_integration_test.go b/pkg/detectors/shipday/shipday_integration_test.gonew file mode 100644index 000000000..0efc9c528--- /dev/null+++ b/pkg/detectors/shipday/shipday_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package shipday++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShipday_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHIPDAY")+ inactiveSecret := testSecrets.MustGetField("SHIPDAY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shipday secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shipday,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shipday secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shipday,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Shipday.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Shipday.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shipday/shipday_test.go b/pkg/detectors/shipday/shipday_test.goindex c6cc50bf3..2dfc61115 100644--- a/pkg/detectors/shipday/shipday_test.go+++ b/pkg/detectors/shipday/shipday_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package shipdayimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShipday_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHIPDAY")- inactiveSecret := testSecrets.MustGetField("SHIPDAY_INACTIVE")+var (+ validPattern = "3ajLBL6g6oj2rC9u2vtrkPt8qSTLp07"+ invalidPattern = "3ajLBL6g6oj2rC9?2vtrkPt8qSTLp07"+ keyword = "shipday"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShipday_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shipday secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shipday,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword shipday",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shipday secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shipday,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Shipday.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Shipday.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shodankey/shodankey_integration_test.go b/pkg/detectors/shodankey/shodankey_integration_test.gonew file mode 100644index 000000000..d9c730cee--- /dev/null+++ b/pkg/detectors/shodankey/shodankey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package shodankey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShodanKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHODANKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SHODANKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shodankey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ShodanKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shodankey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ShodanKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ShodanKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ShodanKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shodankey/shodankey_test.go b/pkg/detectors/shodankey/shodankey_test.goindex d9c730cee..c6ffa6497 100644--- a/pkg/detectors/shodankey/shodankey_test.go+++ b/pkg/detectors/shodankey/shodankey_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package shodankeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShodanKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHODANKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("SHODANKEY_INACTIVE")+var (+ validPattern = "28tEbDdpkFvftp9XkMzAf7O7uVTYlPYD"+ invalidPattern = "28tEbDdp?Fvftp9XkMzAf7O7uVTYlPYD"+ keyword = "shodankey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShodanKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shodankey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ShodanKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword shodankey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shodankey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ShodanKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ShodanKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ShodanKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shopify/shopify_integration_test.go b/pkg/detectors/shopify/shopify_integration_test.gonew file mode 100644index 000000000..0b811f3c9--- /dev/null+++ b/pkg/detectors/shopify/shopify_integration_test.go@@ -0,0 +1,125 @@+package shopify++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShopify_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET")+ inactiveSecret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET_INACTIVE")+ domain := testSecrets.MustGetField("SHOPIFY_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shopify,+ Redacted: domain,+ Verified: true,+ ExtraData: map[string]string{+ "access_scopes": "read_analytics,read_apps,write_assigned_fulfillment_orders,read_assigned_fulfillment_orders,write_customers,read_customers,write_discovery,read_discovery,write_merchant_managed_fulfillment_orders,read_merchant_managed_fulfillment_orders,write_reports,read_reports,write_cart_transforms,read_cart_transforms",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shopify,+ Redacted: domain,+ Verified: false,+ ExtraData: nil,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Shopify.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ got[i].AnalysisInfo = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shopify/shopify_test.go b/pkg/detectors/shopify/shopify_test.goindex f12ae8976..6936375ff 100644--- a/pkg/detectors/shopify/shopify_test.go+++ b/pkg/detectors/shopify/shopify_test.go@@ -4,122 +4,80 @@ import ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShopify_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET")- inactiveSecret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET_INACTIVE")- domain := testSecrets.MustGetField("SHOPIFY_DOMAIN")+var (+ validKey = "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"+ invalidKey = "shpat_?23eEA6cA03dA2eF5CFe9002Be9F1BD9"+ validDomain = "phfjg0Fv-Nn52QAMfsb4LfI7gVp7j-J11WykWugPHRhh3n6lGz8eP0.myshopify.com"+ invalidDomain = "phfjg0Fv-Nn52QAMfsb4LfI7gVp7j-J11W?kWugPHRhh3n6lGz8eP0.myshopify.com"+ keyword = "shopify"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShopify_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shopify,- Redacted: domain,- Verified: true,- ExtraData: map[string]string{- "access_scopes": "read_analytics,read_apps,write_assigned_fulfillment_orders,read_assigned_fulfillment_orders,write_customers,read_customers,write_discovery,read_discovery,write_merchant_managed_fulfillment_orders,read_merchant_managed_fulfillment_orders,write_reports,read_reports,write_cart_transforms,read_cart_transforms",- },- },- },- wantErr: false,+ name: "valid pattern - with keyword shopify",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),+ want: []string{validKey + validDomain},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shopify,- Redacted: domain,- Verified: false,- ExtraData: nil,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Shopify.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil- got[i].AnalysisInfo = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shortcut/shortcut_integration_test.go b/pkg/detectors/shortcut/shortcut_integration_test.gonew file mode 100644index 000000000..c477270db--- /dev/null+++ b/pkg/detectors/shortcut/shortcut_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package shortcut++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShortcut_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHORTCUT")+ inactiveSecret := testSecrets.MustGetField("SHORTCUT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shortcut secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shortcut,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shortcut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shortcut,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Shortcut.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Shortcut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shortcut/shortcut_test.go b/pkg/detectors/shortcut/shortcut_test.goindex c477270db..567872fd5 100644--- a/pkg/detectors/shortcut/shortcut_test.go+++ b/pkg/detectors/shortcut/shortcut_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package shortcutimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShortcut_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHORTCUT")- inactiveSecret := testSecrets.MustGetField("SHORTCUT_INACTIVE")+var (+ validPattern = "2e15518b-b3b1-5269-3746-4dd87951caa2"+ invalidPattern = "2e15518b?b3b1-5269-3746-4dd87951caa2"+ keyword = "shortcut"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShortcut_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shortcut secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shortcut,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword shortcut",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shortcut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shortcut,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Shortcut.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Shortcut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shotstack/shotstack_integration_test.go b/pkg/detectors/shotstack/shotstack_integration_test.gonew file mode 100644index 000000000..5d4afd1a2--- /dev/null+++ b/pkg/detectors/shotstack/shotstack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package shotstack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShotstack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHOTSTACK")+ inactiveSecret := testSecrets.MustGetField("SHOTSTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shotstack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shotstack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shotstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shotstack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Shotstack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Shotstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shotstack/shotstack_test.go b/pkg/detectors/shotstack/shotstack_test.goindex 5a2ec0483..a4060c58a 100644--- a/pkg/detectors/shotstack/shotstack_test.go+++ b/pkg/detectors/shotstack/shotstack_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package shotstackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShotstack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHOTSTACK")- inactiveSecret := testSecrets.MustGetField("SHOTSTACK_INACTIVE")+var (+ validPattern = "3gGdjBjen4jcmMPJa2RJfg5suw47SHihtAznQd4X"+ invalidPattern = "3gGdjBjen4jcmMPJa2RJ?g5suw47SHihtAznQd4X"+ keyword = "shotstack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShotstack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shotstack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shotstack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword shotstack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shotstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shotstack,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Shotstack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Shotstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shutterstock/shutterstock_integration_test.go b/pkg/detectors/shutterstock/shutterstock_integration_test.gonew file mode 100644index 000000000..f8df34474--- /dev/null+++ b/pkg/detectors/shutterstock/shutterstock_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package shutterstock++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShutterstock_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ key := testSecrets.MustGetField("SHUTTERSTOCK_KEY")+ inactiveKey := testSecrets.MustGetField("SHUTTERSTOCK_KEY_INACTIVE")+ secret := testSecrets.MustGetField("SHUTTERSTOCK")+ inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within", key, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shutterstock,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Shutterstock,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Shutterstock.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Shutterstock.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shutterstock/shutterstock_test.go b/pkg/detectors/shutterstock/shutterstock_test.goindex 00fbe034f..7869ef608 100644--- a/pkg/detectors/shutterstock/shutterstock_test.go+++ b/pkg/detectors/shutterstock/shutterstock_test.go@@ -1,122 +1,83 @@-//go:build detectors-// +build detectors-package shutterstockimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShutterstock_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- key := testSecrets.MustGetField("SHUTTERSTOCK_KEY")- inactiveKey := testSecrets.MustGetField("SHUTTERSTOCK_KEY_INACTIVE")- secret := testSecrets.MustGetField("SHUTTERSTOCK")- inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCK_INACTIVE")+var (+ validKey = "35rqejvHx8E83LC82j65uBHEz8VRlWSw"+ invalidKey = "35rqejvHx8E83LC8?j65uBHEz8VRlWSw"+ validSecret = "IJ9u8C6AXnfTwuLd"+ invalidSecret = "IJ9u8C6A?nfTwuLd"+ keyword = "shutterstock"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShutterstock_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within", key, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shutterstock,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword shutterstock",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),+ want: []string{validSecret},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Shutterstock,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Shutterstock.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Shutterstock.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth_integration_test.go b/pkg/detectors/shutterstockoauth/shutterstockoauth_integration_test.gonew file mode 100644index 000000000..cab496900--- /dev/null+++ b/pkg/detectors/shutterstockoauth/shutterstockoauth_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package shutterstockoauth++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestShutterstockOAuth_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH")+ inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ShutterstockOAuth,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ShutterstockOAuth,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ShutterstockOAuth.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ShutterstockOAuth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.goindex d73d4771f..a79ce7979 100644--- a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go+++ b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package shutterstockoauthimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestShutterstockOAuth_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH")- inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH_INACTIVE")+var (+ validPattern = "v2/G8QwLlOmlwlPzafRJTUNIotsfd3kTTRbtsRldZ3tKdQN98abypvIzPywyqnyAivF9Xev6xhEdJfVIRD8ZVA7YKOjsJ5WNz3QG5XUkw0AiCuBTwDLbeidOMGblASRPwnsGrkfYfN1jX6CPywJsOmfP63zXgBZB6H8iXswjYjj8iMV9NAPQ8OSyDnNuhlX4BGFZ9YdrCJEgzHiavGwteueFTpljPODS8oTXwNtAYV4x06eQ6s4bFAAL2nEzuuSakwnDiJd2wXsXW9cLuMHpawLszYYtDLqApOSyZ33KTfmtrvuWQgTcQv5JfYi93mvCdQS0FJyY5AnKSj9hdvqHf2bsXXK5nEcEN41YcKuU8ADk4uceYwEHWyve4xlIqCpdZRWlif3"+ invalidPattern = "v1/G8QwLlOmlwlPzafRJTUNIotsfd3kTTRbtsRldZ3tKdQN98abypvIzPywyqnyAivF9Xev6xhEdJfVIRD8ZVA7YKOjsJ5WNz3QG5XUkw0AiCuBTwDLbeidOMGblASRPwnsGrkfYfN1jX6CPywJsOmfP63zXgBZB6H8iXswjYjj8iMV9NAPQ8OSyDnNuhlX4BGFZ9YdrCJEgzHiavGwteueFTpljPODS8oTXwNtAYV4x06eQ6s4bFAAL2nEzuuSakwnDiJd2wXsXW9cLuMHpawLszYYtDLqApOSyZ33KTfmtrvuWQgTcQv5JfYi93mvCdQS0FJyY5AnKSj9hdvqHf2bsXXK5nEcEN41YcKuU8ADk4uceYwEHWyve4xlIqCpdZRWlif3"+ keyword = "shutterstockoauth"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestShutterstockOAuth_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ShutterstockOAuth,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword shutterstockoauth",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ShutterstockOAuth,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ShutterstockOAuth.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ShutterstockOAuth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/signable/signable_integration_test.go b/pkg/detectors/signable/signable_integration_test.goindex 42b8d9839..b9d5af5f0 100644--- a/pkg/detectors/signable/signable_integration_test.go+++ b/pkg/detectors/signable/signable_integration_test.go@@ -11,9 +11,8 @@ import ("github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-"github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")diff --git a/pkg/detectors/signalwire/signalwire_integration_test.go b/pkg/detectors/signalwire/signalwire_integration_test.gonew file mode 100644index 000000000..02f776f04--- /dev/null+++ b/pkg/detectors/signalwire/signalwire_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package signalwire++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSignalwire_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("SIGNALWIRE_ID")+ secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN")+ url := testSecrets.MustGetField("SIGNALWIRE_URL")+ inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s", secret, url, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Signalwire,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s but not valid", inactiveSecret, url, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Signalwire,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Signalwire.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Signalwire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.goindex a4c06f265..b4f373916 100644--- a/pkg/detectors/signalwire/signalwire_test.go+++ b/pkg/detectors/signalwire/signalwire_test.go@@ -1,122 +1,85 @@-//go:build detectors-// +build detectors-package signalwireimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSignalwire_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("SIGNALWIRE_ID")- secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN")- url := testSecrets.MustGetField("SIGNALWIRE_URL")- inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE")+var (+ validKey = "rBqctDvrJyZ1MAj2aKHpi6wGcPiCzGtEbCSqtE46RB7yVCywVA"+ invalidKey = "rBqctDvrJyZ1MAj2aKHpi6wGc?iCzGtEbCSqtE46RB7yVCywVA"+ validId = "dy0wgsgd-byq5-iqfb-ez1d-dvjepkyub9n9"+ invalidId = "dy0wgsgd?byq5-iqfb-ez1d-dvjepkyub9n9"+ validUrl = "09ft358n4gpa1c.signalwire.com"+ invalidUrl = "09ft358n4gpa1c?signalwire.com"+ keyword = "signalwire"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSignalwire_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s", secret, url, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Signalwire,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword signalwire",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId, keyword, validUrl),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s but not valid", inactiveSecret, url, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Signalwire,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId, keyword, invalidUrl),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Signalwire.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Signalwire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/signaturit/signaturit_integration_test.go b/pkg/detectors/signaturit/signaturit_integration_test.gonew file mode 100644index 000000000..b6670237d--- /dev/null+++ b/pkg/detectors/signaturit/signaturit_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package signaturit++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSignaturit_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIGNATURIT")+ inactiveSecret := testSecrets.MustGetField("SIGNATURIT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a signaturit secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Signaturit,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a signaturit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Signaturit,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Signaturit.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Signaturit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/signaturit/signaturit_test.go b/pkg/detectors/signaturit/signaturit_test.goindex b706d742e..2175d80af 100644--- a/pkg/detectors/signaturit/signaturit_test.go+++ b/pkg/detectors/signaturit/signaturit_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package signaturitimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSignaturit_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIGNATURIT")- inactiveSecret := testSecrets.MustGetField("SIGNATURIT_INACTIVE")+var (+ validPattern = "4AgnWjwDhcgHULFkDkQCY7bra8XfsV9l72JYqsIAFnE9mEo6VusS5yKBJq3OSTEyJfsPoPhorAUc9yUiCxCIxz"+ invalidPattern = "4AgnWjwDhcgHULFkDkQCY7bra8XfsV?l72JYqsIAFnE9mEo6VusS5yKBJq3OSTEyJfsPoPhorAUc9yUiCxCIxz"+ keyword = "signaturit"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSignaturit_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a signaturit secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Signaturit,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword signaturit",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a signaturit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Signaturit,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Signaturit.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Signaturit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/signupgenius/signupgenius_integration_test.go b/pkg/detectors/signupgenius/signupgenius_integration_test.gonew file mode 100644index 000000000..e74b007c3--- /dev/null+++ b/pkg/detectors/signupgenius/signupgenius_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package signupgenius++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSignupgenius_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIGNUPGENIUS")+ inactiveSecret := testSecrets.MustGetField("SIGNUPGENIUS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Signupgenius,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Signupgenius,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Signupgenius.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Signupgenius.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/signupgenius/signupgenius_test.go b/pkg/detectors/signupgenius/signupgenius_test.goindex ffb74e529..eacba7d8e 100644--- a/pkg/detectors/signupgenius/signupgenius_test.go+++ b/pkg/detectors/signupgenius/signupgenius_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package signupgeniusimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSignupgenius_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIGNUPGENIUS")- inactiveSecret := testSecrets.MustGetField("SIGNUPGENIUS_INACTIVE")+var (+ validPattern = "kARx6uGgPqBkho0jzPNfgnAduN3hDjBY"+ invalidPattern = "kARx6uGgPqBkho0j?PNfgnAduN3hDjBY"+ keyword = "signupgenius"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSignupgenius_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Signupgenius,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword signupgenius",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Signupgenius,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Signupgenius.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Signupgenius.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sigopt/sigopt_integration_test.go b/pkg/detectors/sigopt/sigopt_integration_test.gonew file mode 100644index 000000000..c25a2cc62--- /dev/null+++ b/pkg/detectors/sigopt/sigopt_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package sigopt++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSigopt_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIGOPT")+ inactiveSecret := testSecrets.MustGetField("SIGOPT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sigopt,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sigopt,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sigopt.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sigopt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sigopt/sigopt_test.go b/pkg/detectors/sigopt/sigopt_test.goindex c461d56d0..a0053185f 100644--- a/pkg/detectors/sigopt/sigopt_test.go+++ b/pkg/detectors/sigopt/sigopt_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package sigoptimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSigopt_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIGOPT")- inactiveSecret := testSecrets.MustGetField("SIGOPT_INACTIVE")+var (+ validPattern = "GF67UZ5P7UX2A6ELCAE11YDCOYJABCBDYK26O76HRIPU294T"+ invalidPattern = "GF67UZ5P7UX2A6ELCAE11YDC?YJABCBDYK26O76HRIPU294T"+ keyword = "sigopt"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSigopt_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sigopt secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sigopt,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sigopt",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sigopt secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sigopt,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sigopt.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Sigopt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/simfin/simfin_integration_test.go b/pkg/detectors/simfin/simfin_integration_test.gonew file mode 100644index 000000000..999febb9d--- /dev/null+++ b/pkg/detectors/simfin/simfin_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package simfin++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSimFin_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIMFIN")+ inactiveSecret := testSecrets.MustGetField("SIMFIN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simfin secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SimFin,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simfin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SimFin,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SimFin.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SimFin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/simfin/simfin_test.go b/pkg/detectors/simfin/simfin_test.goindex 7ee417e72..66972104b 100644--- a/pkg/detectors/simfin/simfin_test.go+++ b/pkg/detectors/simfin/simfin_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package simfinimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSimFin_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIMFIN")- inactiveSecret := testSecrets.MustGetField("SIMFIN_INACTIVE")+var (+ validPattern = "Y28WVaw01VjEEZQ6zn4HvmfpEWT70nPC"+ invalidPattern = "Y28WVaw01VjEEZQ6?n4HvmfpEWT70nPC"+ keyword = "simfin"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSimFin_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simfin secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SimFin,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword simfin",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simfin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SimFin,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SimFin.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SimFin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/simplesat/simplesat_integration_test.go b/pkg/detectors/simplesat/simplesat_integration_test.gonew file mode 100644index 000000000..e00c0f31b--- /dev/null+++ b/pkg/detectors/simplesat/simplesat_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package simplesat++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSimplesat_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIMPLESAT_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SIMPLESAT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simplesat secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Simplesat,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simplesat secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Simplesat,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Simplesat.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Simplesat.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/simplesat/simplesat_test.go b/pkg/detectors/simplesat/simplesat_test.goindex e00c0f31b..89f4291d9 100644--- a/pkg/detectors/simplesat/simplesat_test.go+++ b/pkg/detectors/simplesat/simplesat_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package simplesatimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSimplesat_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIMPLESAT_TOKEN")- inactiveSecret := testSecrets.MustGetField("SIMPLESAT_INACTIVE")+var (+ validPattern = "52dfk83ssx3hnz3ke35jqqucwy5ojdtyd0at5v4c"+ invalidPattern = "52dfk83ssx3hnz3ke35j?qucwy5ojdtyd0at5v4c"+ keyword = "simplesat"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSimplesat_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simplesat secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Simplesat,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword simplesat",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simplesat secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Simplesat,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Simplesat.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Simplesat.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/simplynoted/simplynoted_integration_test.go b/pkg/detectors/simplynoted/simplynoted_integration_test.gonew file mode 100644index 000000000..dff91caa7--- /dev/null+++ b/pkg/detectors/simplynoted/simplynoted_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package simplynoted++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSimplyNoted_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIMPLYNOTED")+ inactiveSecret := testSecrets.MustGetField("SIMPLYNOTED_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SimplyNoted,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SimplyNoted,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SimplyNoted.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SimplyNoted.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/simplynoted/simplynoted_test.go b/pkg/detectors/simplynoted/simplynoted_test.goindex aaf8f3ec8..0e7f3087f 100644--- a/pkg/detectors/simplynoted/simplynoted_test.go+++ b/pkg/detectors/simplynoted/simplynoted_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package simplynotedimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSimplyNoted_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIMPLYNOTED")- inactiveSecret := testSecrets.MustGetField("SIMPLYNOTED_INACTIVE")+var (+ validPattern = "NSPwR0>Q,wTEJcGyFF)bvY#FACMwu9DzLFD|bLC@D2d1-zILY&sPo6N_AhOwD(:N0`Td[NL2dZt;>yzn3yN'5x[ia&0v2&M0D_r0!L3#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGaX1WAnoZy2Ny20rnl^f8P/Y.u`jxsOy%2iesTS&u|7SMQ]wJ*f2c?lQ:o4&X(/[y.ZK%2Av100u($ZeTU2N4yCFgKp5PqicqSkgjIla31uGS0OvpmpSiy@rFvthHA,k&)uRAM6$>#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGJcGyFF)bvY#FACMwu9DzL"+ invalidPattern = "N PwR0>Q,wTEJcGyFF)bvY#FACMwu9DzLFD|bLC@D2d1-zILY&sPo6N_AhOwD(:N0`Td[NL2dZt;>yzn3yN'5x[ia&0v2&M0D_r0!L3#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGaX1WAnoZy2Ny20rnl^f8P/Y.u`jxsOy%2iesTS&u|7SMQ]wJ*f2c?lQ:o4&X(/[y.ZK%2Av100u($ZeTU2N4yCFgKp5PqicqSkgjIla31uGS0OvpmpSiy@rFvthHA,k&)uRAM6$>#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGJcGyFF)bvY#FACMwu9DzL"+ keyword = "simplynoted"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSimplyNoted_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SimplyNoted,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword simplynoted",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SimplyNoted,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SimplyNoted.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SimplyNoted.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/simvoly/simvoly_integration_test.go b/pkg/detectors/simvoly/simvoly_integration_test.gonew file mode 100644index 000000000..af28be91a--- /dev/null+++ b/pkg/detectors/simvoly/simvoly_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package simvoly++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSimvoly_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIMVOLY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SIMVOLY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simvoly secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Simvoly,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a simvoly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Simvoly,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Simvoly.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Simvoly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/simvoly/simvoly_test.go b/pkg/detectors/simvoly/simvoly_test.goindex 99585908c..fc10692dc 100644--- a/pkg/detectors/simvoly/simvoly_test.go+++ b/pkg/detectors/simvoly/simvoly_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package simvolyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSimvoly_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIMVOLY_TOKEN")- inactiveSecret := testSecrets.MustGetField("SIMVOLY_INACTIVE")+var (+ validPattern = "zu0rpr33yvk0c18vs5wzpx2nq9ww1bh2b"+ invalidPattern = "zu0rpr33yvk0c18v?5wzpx2nq9ww1bh2b"+ keyword = "simvoly"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSimvoly_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simvoly secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Simvoly,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword simvoly",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a simvoly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Simvoly,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Simvoly.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Simvoly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sinchmessage/sinchmessage_integration_test.go b/pkg/detectors/sinchmessage/sinchmessage_integration_test.gonew file mode 100644index 000000000..a40178820--- /dev/null+++ b/pkg/detectors/sinchmessage/sinchmessage_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package sinchmessage++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSinchMessage_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SINCHMESSAGE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SINCHMESSAGE_INACTIVE")+ id := testSecrets.MustGetField("SINCHMESSAGE_PLANID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within sinchid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SinchMessage,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within but sinchid %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SinchMessage,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SinchMessage.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SinchMessage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sinchmessage/sinchmessage_test.go b/pkg/detectors/sinchmessage/sinchmessage_test.goindex 022f04419..53035bb0a 100644--- a/pkg/detectors/sinchmessage/sinchmessage_test.go+++ b/pkg/detectors/sinchmessage/sinchmessage_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package sinchmessageimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSinchMessage_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SINCHMESSAGE_TOKEN")- inactiveSecret := testSecrets.MustGetField("SINCHMESSAGE_INACTIVE")- id := testSecrets.MustGetField("SINCHMESSAGE_PLANID")+var (+ validKey = "k255nuzmx8fzlkjfjz5unlrdbd658bon"+ invalidKey = "k255nuzmx8fzlk?fjz5unlrdbd658bon"+ validId = "y9yvuc6hl2up5qyohet77j23ntll7pr7"+ invalidId = "y9yvuc6?l2up5qyohet77j23ntll7pr7"+ keyword = "sinchmessage"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSinchMessage_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within sinchid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SinchMessage,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sinchmessage",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey, validId, validKey, validId},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within but sinchid %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SinchMessage,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SinchMessage.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SinchMessage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sirv/sirv_integration_test.go b/pkg/detectors/sirv/sirv_integration_test.gonew file mode 100644index 000000000..1905f6a4f--- /dev/null+++ b/pkg/detectors/sirv/sirv_integration_test.go@@ -0,0 +1,118 @@+//go:build detectors+// +build detectors++package sirv++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSirv_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SIRV")+ id := testSecrets.MustGetField("SIRV_ID")+ inactiveSecret := testSecrets.MustGetField("SIRV_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sirv,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sirv,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sirv.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sirv.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/sirv/sirv_test.go b/pkg/detectors/sirv/sirv_test.goindex 1905f6a4f..3b17bc099 100644--- a/pkg/detectors/sirv/sirv_test.go+++ b/pkg/detectors/sirv/sirv_test.go@@ -1,117 +1,82 @@-//go:build detectors-// +build detectors-package sirvimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSirv_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SIRV")- id := testSecrets.MustGetField("SIRV_ID")- inactiveSecret := testSecrets.MustGetField("SIRV_INACTIVE")+var (+ validKey = "Ivus86]AJDk:rghv;8,o00SVP0efds5BBh453jew~mnC$K8elZI-OGU^v0AA#`>vek1cKlkipm]!n/f1xTDlUn1D"+ invalidKey = "?vus86]AJDk:rghv;8,o00SVP0efds5BBh453jew~mnC$K8elZI-OGU^v0AA#`>vek1cKlkipm]!n/f1xTDlUn1D"+ validId = "n64cjK3Dg9Xyr2icHYypLwXl6V"+ invalidId = "n64cjK3Dg9Xyr?icHYypLwXl6V"+ keyword = "sirv"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSirv_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sirv,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sirv",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sirv,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sirv.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Sirv.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/siteleaf/siteleaf_integration_test.go b/pkg/detectors/siteleaf/siteleaf_integration_test.gonew file mode 100644index 000000000..e55d124a5--- /dev/null+++ b/pkg/detectors/siteleaf/siteleaf_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package siteleaf++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSiteleaf_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SITELEAF_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SITELEAF_INACTIVE")+ id := testSecrets.MustGetField("SITELEAF_USERID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Siteleaf,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Siteleaf,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Siteleaf.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Siteleaf.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/siteleaf/siteleaf_test.go b/pkg/detectors/siteleaf/siteleaf_test.goindex 64da43b06..1f1f716b9 100644--- a/pkg/detectors/siteleaf/siteleaf_test.go+++ b/pkg/detectors/siteleaf/siteleaf_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package siteleafimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSiteleaf_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SITELEAF_TOKEN")- inactiveSecret := testSecrets.MustGetField("SITELEAF_INACTIVE")- id := testSecrets.MustGetField("SITELEAF_USERID")+var (+ validKey = "lklep3linr2itd42nkncw9b8s9g1yrgb"+ invalidKey = "lklep3li?r2itd42nkncw9b8s9g1yrgb"+ validId = "zoc07j72vs6q1lxs6j5vkzz7mefhiii6"+ invalidId = "zoc07j72vs?q1lxs6j5vkzz7mefhiii6"+ keyword = "siteleaf"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSiteleaf_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Siteleaf,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword siteleaf",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey, validId, validKey, validId},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Siteleaf,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Siteleaf.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Siteleaf.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/skrappio/skrappio_integration_test.go b/pkg/detectors/skrappio/skrappio_integration_test.gonew file mode 100644index 000000000..709f782f7--- /dev/null+++ b/pkg/detectors/skrappio/skrappio_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package skrappio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSkrapio_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SKRAPIO_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SKRAPIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a skrapp secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Skrappio,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a skrapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Skrappio,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Skrapio.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Skrapio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/skrappio/skrappio_test.go b/pkg/detectors/skrappio/skrappio_test.goindex a1520533d..ee8222cfd 100644--- a/pkg/detectors/skrappio/skrappio_test.go+++ b/pkg/detectors/skrappio/skrappio_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package skrappioimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSkrapio_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SKRAPIO_TOKEN")- inactiveSecret := testSecrets.MustGetField("SKRAPIO_INACTIVE")+var (+ validPattern = "rAheANMWIEHz9GfmtocXjvG8ESfdZvgSMTbDBIR7SA"+ invalidPattern = "rAheANMWIEHz9GfmtocXj?G8ESfdZvgSMTbDBIR7SA"+ keyword = "skrappio"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSkrapio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a skrapp secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Skrappio,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword skrappio",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a skrapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Skrappio,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Skrapio.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Skrapio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/skybiometry/skybiometry_integration_test.go b/pkg/detectors/skybiometry/skybiometry_integration_test.gonew file mode 100644index 000000000..f7d96e29b--- /dev/null+++ b/pkg/detectors/skybiometry/skybiometry_integration_test.go@@ -0,0 +1,130 @@+//go:build detectors+// +build detectors++package skybiometry++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSkyBiometry_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ key := testSecrets.MustGetField("SKYBIOMETRY_KEY")+ inactiveKey := testSecrets.MustGetField("SKYBIOMETRY_KEY_INACTIVE")+ secret := testSecrets.MustGetField("SKYBIOMETRY")+ inactiveSecret := testSecrets.MustGetField("SKYBIOMETRY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within", key, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SkyBiometry,+ Verified: true,+ },+ {+ DetectorType: detectorspb.DetectorType_SkyBiometry,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SkyBiometry,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_SkyBiometry,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SkyBiometry.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SkyBiometry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/skybiometry/skybiometry_test.go b/pkg/detectors/skybiometry/skybiometry_test.goindex f7d96e29b..f052f95e0 100644--- a/pkg/detectors/skybiometry/skybiometry_test.go+++ b/pkg/detectors/skybiometry/skybiometry_test.go@@ -1,130 +1,83 @@-//go:build detectors-// +build detectors-package skybiometryimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSkyBiometry_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- key := testSecrets.MustGetField("SKYBIOMETRY_KEY")- inactiveKey := testSecrets.MustGetField("SKYBIOMETRY_KEY_INACTIVE")- secret := testSecrets.MustGetField("SKYBIOMETRY")- inactiveSecret := testSecrets.MustGetField("SKYBIOMETRY_INACTIVE")+var (+ validKey = "qdmxigxqpkpo8i38zxfuxct3d2"+ invalidKey = "qdmx?gxqpkpo8i38zxfuxct3d2"+ validSecret = "ko86ttplsbrao397eub88pqniu"+ invalidSecret = "ko86ttplsbrao?97eub88pqniu"+ keyword = "skybiometry"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSkyBiometry_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within", key, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SkyBiometry,- Verified: true,- },- {- DetectorType: detectorspb.DetectorType_SkyBiometry,- Verified: false,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SkyBiometry,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_SkyBiometry,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword skybiometry",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),+ want: []string{validKey, validSecret},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SkyBiometry.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SkyBiometry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/slack/slack_integration_test.go b/pkg/detectors/slack/slack_integration_test.gonew file mode 100644index 000000000..19d696753--- /dev/null+++ b/pkg/detectors/slack/slack_integration_test.go@@ -0,0 +1,189 @@+//go:build detectors+// +build detectors++package slack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSlack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SLACK")+ secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }++ tests := []struct {+ name string+ s Scanner+ args args+ wantResults []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),+ verify: true,+ },+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: true,+ ExtraData: map[string]string{+ "name": "marge.haskell.bridge",+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",+ "team": "ct.org",+ "token_type": "Slack User Token",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found but unverified",+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),+ verify: true,+ },+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: false,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",+ "token_type": "Slack User Token",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "account_inactive",+ s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "account_inactive"}`)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),+ verify: true,+ },+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: false,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",+ "token_type": "Slack User Token",+ },+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),+ verify: true,+ },+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: false,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",+ "token_type": "Slack User Token",+ },+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "unexpected auth response",+ s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),+ verify: true,+ },+ wantResults: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Slack,+ Verified: false,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",+ "token_type": "Slack User Token",+ },+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }++ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ got[i].AnalysisInfo = nil++ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {+ t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/slack/slack_test.go b/pkg/detectors/slack/slack_test.goindex 19d696753..49f8e3c2e 100644--- a/pkg/detectors/slack/slack_test.go+++ b/pkg/detectors/slack/slack_test.go@@ -1,188 +1,86 @@-//go:build detectors-// +build detectors-package slackimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSlack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SLACK")- secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validBotToken = "xoxb-65677559833-9613778673399u"+ invalidBotToken = "xoxb-65677559833?9613778673399u"+ validUserToken = "xoxp-24232636415-0285024315463c"+ invalidUserToken = "xoxp-24232636415?0285024315463c"+ validWorkspaceAccessToken = "xoxa-08532509747-07570405353c"+ invalidWorkspaceAccessToken = "xoxa-08532509747?07570405353c"+ validWorkspaceRefreshToken = "xoxr-833485595373-24619897332l"+ invalidWorkspaceRefreshToken = "xoxr-833485595373?24619897332l"+ keyword = "slack"+)+func TestSlack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- wantResults []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),- verify: true,- },- wantResults: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Slack,- Verified: true,- ExtraData: map[string]string{- "name": "marge.haskell.bridge",- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",- "team": "ct.org",- "token_type": "Slack User Token",- },- },- },- wantErr: false,- },- {- name: "found but unverified",- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),- verify: true,- },- wantResults: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Slack,- Verified: false,- ExtraData: map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",- "token_type": "Slack User Token",- },- },- },- wantErr: false,- },- {- name: "account_inactive",- s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "account_inactive"}`)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),- verify: true,- },- wantResults: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Slack,- Verified: false,- ExtraData: map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",- "token_type": "Slack User Token",- },- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),- verify: true,- },- wantResults: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Slack,- Verified: false,- ExtraData: map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",- "token_type": "Slack User Token",- },- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - with keyword slack",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validBotToken, keyword, validUserToken, keyword, validWorkspaceAccessToken, keyword, validWorkspaceRefreshToken),+ want: []string{validBotToken, validUserToken, validWorkspaceAccessToken, validWorkspaceRefreshToken},},{- name: "unexpected auth response",- s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),- verify: true,- },- wantResults: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Slack,- Verified: false,- ExtraData: map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",- "token_type": "Slack User Token",- },- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidBotToken, keyword, invalidUserToken, keyword, invalidWorkspaceAccessToken, keyword, invalidWorkspaceRefreshToken),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")+ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil- got[i].AnalysisInfo = nil+ return+ }- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {- t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/smartsheets/smartsheets_integration_test.go b/pkg/detectors/smartsheets/smartsheets_integration_test.gonew file mode 100644index 000000000..386ec16e9--- /dev/null+++ b/pkg/detectors/smartsheets/smartsheets_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package smartsheets++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSmartsheets_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SMARTSHEETS_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SMARTSHEETS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Smartsheets,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Smartsheets,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Smartsheets.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Smartsheets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/smartsheets/smartsheets_test.go b/pkg/detectors/smartsheets/smartsheets_test.goindex 81facebc8..a388b7304 100644--- a/pkg/detectors/smartsheets/smartsheets_test.go+++ b/pkg/detectors/smartsheets/smartsheets_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package smartsheetsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSmartsheets_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SMARTSHEETS_TOKEN")- inactiveSecret := testSecrets.MustGetField("SMARTSHEETS_INACTIVE")+var (+ validPattern = "MVE7zmdxouvunYkowLzaudyX7tvMpkqJ3q52C"+ invalidPattern = "MVE7?mdxouvunYkowLzaudyX7tvMpkqJ3q52C"+ keyword = "smartsheets"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSmartsheets_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Smartsheets,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword smartsheets",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Smartsheets,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Smartsheets.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Smartsheets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/smartystreets/smartystreets_integration_test.go b/pkg/detectors/smartystreets/smartystreets_integration_test.gonew file mode 100644index 000000000..a4a34abbe--- /dev/null+++ b/pkg/detectors/smartystreets/smartystreets_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package smartystreets++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSmartyStreets_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SMARTYSTREETS")+ id := testSecrets.MustGetField("SMARTYSTREETS_ID")+ inactiveSecret := testSecrets.MustGetField("SMARTYSTREETS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SmartyStreets,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SmartyStreets,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SmartyStreets.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SmartyStreets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/smartystreets/smartystreets_test.go b/pkg/detectors/smartystreets/smartystreets_test.goindex b293f5914..cd04045f5 100644--- a/pkg/detectors/smartystreets/smartystreets_test.go+++ b/pkg/detectors/smartystreets/smartystreets_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package smartystreetsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSmartyStreets_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SMARTYSTREETS")- id := testSecrets.MustGetField("SMARTYSTREETS_ID")- inactiveSecret := testSecrets.MustGetField("SMARTYSTREETS_INACTIVE")+var (+ validKey = "UzwkWoJlOqT5q9pXbX2n"+ invalidKey = "UzwkWoJlOq?5q9pXbX2n"+ validId = "pers5g2aa5icwflc1cmqjrs3ij8iqjmsg1x3"+ invalidId = "pers5g2aa5icwflc1c?qjrs3ij8iqjmsg1x3"+ keyword = "smartystreets"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSmartyStreets_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SmartyStreets,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword smartystreets",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SmartyStreets,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SmartyStreets.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SmartyStreets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/smooch/smooch_integration_test.go b/pkg/detectors/smooch/smooch_integration_test.gonew file mode 100644index 000000000..b982b5213--- /dev/null+++ b/pkg/detectors/smooch/smooch_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package smooch++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSmooch_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SMOOCH_SECRET")+ id := testSecrets.MustGetField("SMOOCH_ID")+ inactiveID := testSecrets.MustGetField("SMOOCH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Smooch,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Smooch,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Smooch.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Smooch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/smooch/smooch_test.go b/pkg/detectors/smooch/smooch_test.goindex 7c5030306..a68cd36f5 100644--- a/pkg/detectors/smooch/smooch_test.go+++ b/pkg/detectors/smooch/smooch_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package smoochimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSmooch_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SMOOCH_SECRET")- id := testSecrets.MustGetField("SMOOCH_ID")- inactiveID := testSecrets.MustGetField("SMOOCH_INACTIVE")+var (+ validKey = "act_svf99w1upfk2syajjx6wxxg9"+ invalidKey = "act_svf99w1upf?2syajjx6wxxg9"+ validSecret = "w2XXiJgDfMJnkKQMPKEF4AzeHnQseEcIsK0C7Gm0icA5PtRM4GIZl82B84n01RJMmkz3aGHtpP_44UXXKhW76L"+ invalidSecret = "w2XXiJgDfMJnkKQMPKEF4AzeHnQseEcIsK0C7Gm0icA?PtRM4GIZl82B84n01RJMmkz3aGHtpP_44UXXKhW76L"+ keyword = "smooch"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSmooch_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Smooch,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword smooch",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),+ want: []string{validKey + validSecret},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Smooch,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Smooch.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Smooch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/snipcart/snipcart_integration_test.go b/pkg/detectors/snipcart/snipcart_integration_test.gonew file mode 100644index 000000000..35be88a33--- /dev/null+++ b/pkg/detectors/snipcart/snipcart_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package snipcart++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSnipcart_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SNIPCART")+ inactiveSecret := testSecrets.MustGetField("SNIPCART_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a snipcart secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Snipcart,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a snipcart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Snipcart,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Snipcart.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Snipcart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/snipcart/snipcart_test.go b/pkg/detectors/snipcart/snipcart_test.goindex be3328a7f..a8a754fb8 100644--- a/pkg/detectors/snipcart/snipcart_test.go+++ b/pkg/detectors/snipcart/snipcart_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package snipcartimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSnipcart_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SNIPCART")- inactiveSecret := testSecrets.MustGetField("SNIPCART_INACTIVE")+var (+ validPattern = "zc62fqbZm4zyVsdPFAcaRhFnSFpO12Lpmas3PU9IK5CJEklhJDeybwRr8c7ibBG3fL3iWoXTHbb"+ invalidPattern = "zc62fqbZm4zyVsdPFAcaRhFnSFpO12Lpmas3P?9IK5CJEklhJDeybwRr8c7ibBG3fL3iWoXTHbb"+ keyword = "snipcart"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSnipcart_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a snipcart secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Snipcart,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword snipcart",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a snipcart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Snipcart,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Snipcart.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Snipcart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/snowflake/snowflake_integration_test.go b/pkg/detectors/snowflake/snowflake_integration_test.gonew file mode 100644index 000000000..d7d7cdbe9--- /dev/null+++ b/pkg/detectors/snowflake/snowflake_integration_test.go@@ -0,0 +1,175 @@+//go:build detectors+// +build detectors++package snowflake++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSnowflake_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ accountIdentifier := testSecrets.MustGetField("SNOWFLAKE_ACCOUNT")+ username := testSecrets.MustGetField("SNOWFLAKE_USERNAME")+ password := testSecrets.MustGetField("SNOWFLAKE_PASS")+ inactivePassword := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE")++ // Create a context with a past deadline to simulate DeadlineExceeded error+ pastTime := time.Now().Add(-time.Second) // Set the deadline in the past+ errorCtx, cancel := context.WithDeadline(context.Background(), pastTime)+ defer cancel()++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Snowflake,+ Verified: true,+ ExtraData: map[string]string{+ "account": accountIdentifier,+ "username": username,+ },+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, inactivePassword)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Snowflake,+ Verified: false,+ ExtraData: map[string]string{+ "account": accountIdentifier,+ "username": username,+ },+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, indeterminate error (timeout)",+ s: Scanner{},+ args: args{+ ctx: errorCtx,+ data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Snowflake,+ ExtraData: map[string]string{+ "account": accountIdentifier,+ "username": username,+ },+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Snowflake.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ keysToCopy := []string{"account", "username"}+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }++ got[i].ExtraData = newMap(got[i].ExtraData, keysToCopy)+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Snowflake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func newMap(extraMap map[string]string, keysToCopy []string) map[string]string {+ newExtraDataMap := make(map[string]string)+ for _, key := range keysToCopy {+ if value, ok := extraMap[key]; ok {+ newExtraDataMap[key] = value+ }+ }+ return newExtraDataMap+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/snowflake/snowflake_test.go b/pkg/detectors/snowflake/snowflake_test.goindex f06a872fd..45acafa11 100644--- a/pkg/detectors/snowflake/snowflake_test.go+++ b/pkg/detectors/snowflake/snowflake_test.go@@ -1,21 +1,14 @@-//go:build detectors-// +build detectors-package snowflakeimport ("context""fmt""testing"- "time""github.com/brianvoe/gofakeit/v7""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")func TestSnowflake_Pattern(t *testing.T) {@@ -104,160 +97,3 @@ func TestSnowflake_Pattern(t *testing.T) {})}}--func TestSnowflake_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }-- accountIdentifier := testSecrets.MustGetField("SNOWFLAKE_ACCOUNT")- username := testSecrets.MustGetField("SNOWFLAKE_USERNAME")- password := testSecrets.MustGetField("SNOWFLAKE_PASS")- inactivePassword := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE")-- // Create a context with a past deadline to simulate DeadlineExceeded error- pastTime := time.Now().Add(-time.Second) // Set the deadline in the past- errorCtx, cancel := context.WithDeadline(context.Background(), pastTime)- defer cancel()-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Snowflake,- Verified: true,- ExtraData: map[string]string{- "account": accountIdentifier,- "username": username,- },- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, inactivePassword)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Snowflake,- Verified: false,- ExtraData: map[string]string{- "account": accountIdentifier,- "username": username,- },- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, indeterminate error (timeout)",- s: Scanner{},- args: args{- ctx: errorCtx,- data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Snowflake,- ExtraData: map[string]string{- "account": accountIdentifier,- "username": username,- },- },- },- wantErr: false,- wantVerificationErr: true,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Snowflake.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- keysToCopy := []string{"account", "username"}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())- }-- got[i].ExtraData = newMap(got[i].ExtraData, keysToCopy)- }- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Snowflake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func newMap(extraMap map[string]string, keysToCopy []string) map[string]string {- newExtraDataMap := make(map[string]string)- for _, key := range keysToCopy {- if value, ok := extraMap[key]; ok {- newExtraDataMap[key] = value- }- }- return newExtraDataMap-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }- }- })- }-}diff --git a/pkg/detectors/sonarcloud/sonarcloud_integration_test.go b/pkg/detectors/sonarcloud/sonarcloud_integration_test.gonew file mode 100644index 000000000..fb9554885--- /dev/null+++ b/pkg/detectors/sonarcloud/sonarcloud_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package sonarcloud++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSonarCloud_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SONARCLOUD")+ inactiveSecret := testSecrets.MustGetField("SONARCLOUD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SonarCloud,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SonarCloud,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SonarCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SonarCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sonarcloud/sonarcloud_test.go b/pkg/detectors/sonarcloud/sonarcloud_test.goindex 2d1720b2c..142b34352 100644--- a/pkg/detectors/sonarcloud/sonarcloud_test.go+++ b/pkg/detectors/sonarcloud/sonarcloud_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package sonarcloudimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSonarCloud_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SONARCLOUD")- inactiveSecret := testSecrets.MustGetField("SONARCLOUD_INACTIVE")+var (+ validPattern = "hxrxxgxtcjxj7ta3dn33c5r5i2h0i6cqjv9kwkye"+ invalidPattern = "hxrxxgxt?jxj7ta3dn33c5r5i2h0i6cqjv9kwkye"+ keyword = "sonarcloud"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSonarCloud_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SonarCloud,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sonarcloud",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SonarCloud,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SonarCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SonarCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sourcegraph/sourcegraph_integration_test.go b/pkg/detectors/sourcegraph/sourcegraph_integration_test.gonew file mode 100644index 000000000..fb695ca4e--- /dev/null+++ b/pkg/detectors/sourcegraph/sourcegraph_integration_test.go@@ -0,0 +1,253 @@+//go:build detectors+// +build detectors++package sourcegraph++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSourcegraph_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ secretV1 := testSecrets.MustGetField("SOURCEGRAPH_V1")+ secretV2 := testSecrets.MustGetField("SOURCEGRAPH_V2")+ secretV3 := testSecrets.MustGetField("SOURCEGRAPH_V3")++ inactiveSecretV1 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V1")+ inactiveSecretV2 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V2")+ inactiveSecretV3 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V3")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified v1",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, verified v1",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, verified v2",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV2)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, verified v3",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV3)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified v1",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV1)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified v2",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV2)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified v3",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV3)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sourcegraph,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ got[i].AnalysisInfo = nil+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError", "ExtraData")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sourcegraph/sourcegraph_test.go b/pkg/detectors/sourcegraph/sourcegraph_test.goindex c15c09527..8e7bcf84b 100644--- a/pkg/detectors/sourcegraph/sourcegraph_test.go+++ b/pkg/detectors/sourcegraph/sourcegraph_test.go@@ -1,253 +1,80 @@-//go:build detectors-// +build detectors-package sourcegraphimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSourcegraph_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }-- secretV1 := testSecrets.MustGetField("SOURCEGRAPH_V1")- secretV2 := testSecrets.MustGetField("SOURCEGRAPH_V2")- secretV3 := testSecrets.MustGetField("SOURCEGRAPH_V3")-- inactiveSecretV1 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V1")- inactiveSecretV2 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V2")- inactiveSecretV3 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V3")+var (+ validPattern = "sgp_4a2aD05B07721CE0_EEBdEacB356CdC6Dc8CD317A4276e2cEE9cb118a"+ invalidPattern = "sgp_4a2aD0?B07721CE0_EEBdEacB356CdC6Dc8CD317A4276e2cEE9cb118a"+ keyword = "sourcegraph"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSourcegraph_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified v1",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, verified v1",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword sourcegraph",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, verified v2",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV2)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, verified v3",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV3)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified v1",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV1)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified v2",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV2)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified v3",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV3)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sourcegraph,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}- got[i].AnalysisInfo = nil}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError", "ExtraData")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/sourcegraphcody/sourcegraphcody_integration_test.go b/pkg/detectors/sourcegraphcody/sourcegraphcody_integration_test.gonew file mode 100644index 000000000..824beac3c--- /dev/null+++ b/pkg/detectors/sourcegraphcody/sourcegraphcody_integration_test.go@@ -0,0 +1,161 @@+//go:build detectors+// +build detectors++package sourcegraphcody++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSourcegraphcody_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SOURCEGRAPHCODY")+ inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SourcegraphCody,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SourcegraphCody,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SourcegraphCody,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SourcegraphCody,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go b/pkg/detectors/sourcegraphcody/sourcegraphcody_test.goindex 2445ebfa2..cac9cd42f 100644--- a/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go+++ b/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go@@ -1,161 +1,80 @@-//go:build detectors-// +build detectors-package sourcegraphcodyimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSourcegraphcody_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SOURCEGRAPHCODY")- inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")+var (+ validPattern = "slk_3c6669a93b14fbd6061e00fb40876b327826df4fe393e760d3109de08165e793"+ invalidPattern = "slk_?c6669a93b14fbd6061e00fb40876b327826df4fe393e760d3109de08165e793"+ keyword = "sourcegraphcody"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSourcegraphcody_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SourcegraphCody,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SourcegraphCody,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword sourcegraphcody",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SourcegraphCody,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SourcegraphCody,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/sparkpost/sparkpost_integration_test.go b/pkg/detectors/sparkpost/sparkpost_integration_test.gonew file mode 100644index 000000000..6711fe639--- /dev/null+++ b/pkg/detectors/sparkpost/sparkpost_integration_test.go@@ -0,0 +1,123 @@+//go:build detectors+// +build detectors++package sparkpost++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSparkpost_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SPARKPOST_API_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SPARKPOST_API_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sparkpost,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sparkpost,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)++ if (err != nil) != tt.wantErr {+ t.Errorf("Sparkpost.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }++ if diff := pretty.Compare(got, tt.want); diff != "" {++ t.Errorf("Sparkpost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sparkpost/sparkpost_test.go b/pkg/detectors/sparkpost/sparkpost_test.goindex f82992dda..468120100 100644--- a/pkg/detectors/sparkpost/sparkpost_test.go+++ b/pkg/detectors/sparkpost/sparkpost_test.go@@ -1,123 +1,81 @@-//go:build detectors-// +build detectors-package sparkpostimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSparkpost_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SPARKPOST_API_TOKEN")- inactiveSecret := testSecrets.MustGetField("SPARKPOST_API_INACTIVE")+var (+ validPattern = "VlsweyCt4GRglwZ8G3xVo5lY8Z4Wo9cLxyHFWh9V"+ invalidPattern = "VlsweyCt4GRglwZ8G3xV?5lY8Z4Wo9cLxyHFWh9V"+ keyword = "sparkpost"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSparkpost_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sparkpost,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sparkpost,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword sparkpost",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sparkpost.FromData() error = %v, wantErr %v", err, tt.wantErr)+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {+ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }- t.Errorf("Sparkpost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/speechtextai/speechtextai_integration_test.go b/pkg/detectors/speechtextai/speechtextai_integration_test.gonew file mode 100644index 000000000..f3184a5fd--- /dev/null+++ b/pkg/detectors/speechtextai/speechtextai_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package speechtextai++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSpeechTextAI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SPEECHTEXTAI")+ inactiveSecret := testSecrets.MustGetField("SPEECHTEXTAI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a speechtext secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SpeechTextAI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a speechtext secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SpeechTextAI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SpeechTextAI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SpeechTextAI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/speechtextai/speechtextai_test.go b/pkg/detectors/speechtextai/speechtextai_test.goindex f759c6ce9..6c1ef9183 100644--- a/pkg/detectors/speechtextai/speechtextai_test.go+++ b/pkg/detectors/speechtextai/speechtextai_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package speechtextaiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSpeechTextAI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SPEECHTEXTAI")- inactiveSecret := testSecrets.MustGetField("SPEECHTEXTAI_INACTIVE")+var (+ validPattern = "51bb70d3ab4f9e0ecd17ec0a6f8ed289"+ invalidPattern = "G1bb70d3ab4f9e0ecd17ec0a6f8ed289"+ keyword = "speechtextai"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSpeechTextAI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a speechtext secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SpeechTextAI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword speechtextai",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a speechtext secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SpeechTextAI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SpeechTextAI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SpeechTextAI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_integration_test.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_integration_test.gonew file mode 100644index 000000000..49de9e00a--- /dev/null+++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package splunkobservabilitytoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSplunkObservabilityToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SPLUNK_ACCESS_TOKEN") // TODO: rename+ inactiveSecret := testSecrets.MustGetField("SPLUNK_ACCESS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a splunk secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a splunk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SplunkObservabilityToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SplunkObservabilityToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.goindex 6d172f8fa..9cf7298d1 100644--- a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go+++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package splunkobservabilitytokenimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSplunkObservabilityToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SPLUNK_ACCESS_TOKEN") // TODO: rename- inactiveSecret := testSecrets.MustGetField("SPLUNK_ACCESS_INACTIVE")+var (+ validPattern = "OlKU6qOr3U4AlCziwvFCY2"+ invalidPattern = "OlKU6qOr3U4?lCziwvFCY2"+ keyword = "splunkobservabilitytoken"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSplunkObservabilityToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a splunk secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword splunkobservabilitytoken",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a splunk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SplunkObservabilityToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SplunkObservabilityToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/spoonacular/spoonacular_integration_test.go b/pkg/detectors/spoonacular/spoonacular_integration_test.gonew file mode 100644index 000000000..91ba6069b--- /dev/null+++ b/pkg/detectors/spoonacular/spoonacular_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package spoonacular++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSpoonacular_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SPOONACULAR")+ inactiveSecret := testSecrets.MustGetField("SPOONACULAR_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Spoonacular,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Spoonacular,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Spoonacular.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Spoonacular.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/spoonacular/spoonacular_test.go b/pkg/detectors/spoonacular/spoonacular_test.goindex f4a786aca..d8d5e323b 100644--- a/pkg/detectors/spoonacular/spoonacular_test.go+++ b/pkg/detectors/spoonacular/spoonacular_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package spoonacularimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSpoonacular_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SPOONACULAR")- inactiveSecret := testSecrets.MustGetField("SPOONACULAR_INACTIVE")+var (+ validPattern = "xyclb6zdiq9hyexe74ah4lvp3ea02sbv"+ invalidPattern = "xyclb6zdiq9hyexe?4ah4lvp3ea02sbv"+ keyword = "spoonacular"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSpoonacular_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Spoonacular,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword spoonacular",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Spoonacular,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Spoonacular.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Spoonacular.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sportsmonk/sportsmonk_integration_test.go b/pkg/detectors/sportsmonk/sportsmonk_integration_test.gonew file mode 100644index 000000000..399df68b6--- /dev/null+++ b/pkg/detectors/sportsmonk/sportsmonk_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package sportsmonk++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSportsmonk_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SPORTSMONK")+ inactiveSecret := testSecrets.MustGetField("SPORTSMONK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sportsmonk,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sportsmonk,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sportsmonk.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sportsmonk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sportsmonk/sportsmonk_test.go b/pkg/detectors/sportsmonk/sportsmonk_test.goindex a04837313..4bf2390e4 100644--- a/pkg/detectors/sportsmonk/sportsmonk_test.go+++ b/pkg/detectors/sportsmonk/sportsmonk_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package sportsmonkimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSportsmonk_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SPORTSMONK")- inactiveSecret := testSecrets.MustGetField("SPORTSMONK_INACTIVE")+var (+ validPattern = "GxSYzYakznVR2ZSu7cdIGjj2g3m4VD7vkhToot3WKwEJlgibw4a6akEeV55V"+ invalidPattern = "GxSYzYakznVR2ZSu?cdIGjj2g3m4VD7vkhToot3WKwEJlgibw4a6akEeV55V"+ keyword = "sportsmonk"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSportsmonk_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sportsmonk,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sportsmonk",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sportsmonk,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sportsmonk.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Sportsmonk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/spotifykey/spotifykey_integration_test.go b/pkg/detectors/spotifykey/spotifykey_integration_test.gonew file mode 100644index 000000000..a38741bfa--- /dev/null+++ b/pkg/detectors/spotifykey/spotifykey_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package spotifykey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSpotifyKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ clientID := testSecrets.MustGetField("SPOTIFY_CLIENTID")+ clientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_SECRET")+ inactiveClientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s", clientSecret, clientID)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SpotifyKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s but not valid", inactiveClientSecret, clientID)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SpotifyKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SpotifyKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SpotifyKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/spotifykey/spotifykey_test.go b/pkg/detectors/spotifykey/spotifykey_test.goindex 3aff9fe77..8999a08ba 100644--- a/pkg/detectors/spotifykey/spotifykey_test.go+++ b/pkg/detectors/spotifykey/spotifykey_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package spotifykeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSpotifyKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- clientID := testSecrets.MustGetField("SPOTIFY_CLIENTID")- clientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_SECRET")- inactiveClientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_INACTIVE")+var (+ validSecret = "MJsy6euxw0GKljGYZLZGnn0PvKuZB0V8"+ invalidSecret = "MJsy6euxw0GKljGY?LZGnn0PvKuZB0V8"+ validId = "JykXADk7r8ZegkS0emhtpFSWNqGVAIgm"+ invalidId = "JykXADk7r8Z?gkS0emhtpFSWNqGVAIgm"+ keyword = "spotifykey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSpotifyKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s", clientSecret, clientID)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SpotifyKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword spotifykey",+ input: fmt.Sprintf("%s key - '%s'\n%s id - '%s'\n", keyword, validSecret, keyword, validId),+ want: []string{validSecret, validId},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s but not valid", inactiveClientSecret, clientID)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SpotifyKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s key - '%s'\n%s id - '%s'\n", keyword, invalidSecret, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SpotifyKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SpotifyKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sqlserver/sqlserver_test.go b/pkg/detectors/sqlserver/sqlserver_test.goindex 33b5fffe2..fc5748b71 100644--- a/pkg/detectors/sqlserver/sqlserver_test.go+++ b/pkg/detectors/sqlserver/sqlserver_test.go@@ -4,7 +4,7 @@ import ("testing")-func TestSQLServer_pattern(t *testing.T) {+func TestSQLServer_Pattern(t *testing.T) {if !pattern.Match([]byte(`builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`)) {t.Errorf("SQLServer.pattern: did not find connection string from Program.cs")}diff --git a/pkg/detectors/square/square_integration_test.go b/pkg/detectors/square/square_integration_test.gonew file mode 100644index 000000000..d6b7793d4--- /dev/null+++ b/pkg/detectors/square/square_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package square++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSquare_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SQUARE_SECRET")+ secretInactive := testSecrets.MustGetField("SQUARE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unvierifed",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Square.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ got[i].AnalysisInfo = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Square.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/square/square_test.go b/pkg/detectors/square/square_test.goindex 57d5fa21b..dadda930f 100644--- a/pkg/detectors/square/square_test.go+++ b/pkg/detectors/square/square_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package squareimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSquare_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SQUARE_SECRET")- secretInactive := testSecrets.MustGetField("SQUARE_INACTIVE")+var (+ validPattern = "EAAAYmTgYL4kQ-65xyZ4zANgipYVJQFTrb=roK23I=iFebzL4rjYbUg10N6o1l--"+ invalidPattern = "EAAAYmTgYL4kQ-65xyZ4zANgipYVJQFT?b=roK23I=iFebzL4rjYbUg10N6o1l--"+ keyword = "square"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSquare_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a square secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Square,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword square",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unvierifed",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a square secret %s within", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Square,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Square.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil- got[i].AnalysisInfo = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Square.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/squareapp/squareapp_integration_test.go b/pkg/detectors/squareapp/squareapp_integration_test.gonew file mode 100644index 000000000..e36008c59--- /dev/null+++ b/pkg/detectors/squareapp/squareapp_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package squareapp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSquareApp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SQUAREAPP_SECRET")+ secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")+ id := testSecrets.MustGetField("SQUAREAPP_ID")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: true,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Square,+ Verified: false,+ Redacted: id,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SquareApp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SquareApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/squareapp/squareapp_test.go b/pkg/detectors/squareapp/squareapp_test.goindex ad9874c6f..41918ed8c 100644--- a/pkg/detectors/squareapp/squareapp_test.go+++ b/pkg/detectors/squareapp/squareapp_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package squareappimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSquareApp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SQUAREAPP_SECRET")- secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")- id := testSecrets.MustGetField("SQUAREAPP_ID")- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validKey = "YJsq0ige-a9khwVJOSwlzBvX0wp4j8t90s2d"+ invalidKey = "YJsq0ige-a9kh?VJOSwlzBvX0wp4j8t90s2d"+ validSec = "4sSPeeM_jk0VZiTFZJqEwzvXHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"+ invalidSec = "4sSPeeM_jk0VZiTFZJqEwz?XHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"+ keyword = "squareapp"+)++func TestSquareApp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Square,- Verified: true,- Redacted: id,- },- },- wantErr: false,+ name: "valid pattern - with keyword squareapp",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSec),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Square,- Verified: false,- Redacted: id,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSec),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SquareApp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SquareApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/squarespace/squarespace_integration_test.go b/pkg/detectors/squarespace/squarespace_integration_test.gonew file mode 100644index 000000000..80308cb71--- /dev/null+++ b/pkg/detectors/squarespace/squarespace_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package squarespace++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSquarespace_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SQUARESPACE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SQUARESPACE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squarespace secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Squarespace,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squarespace secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Squarespace,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Squarespace.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Squarespace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/squarespace/squarespace_test.go b/pkg/detectors/squarespace/squarespace_test.goindex 1874e5816..ff66e179a 100644--- a/pkg/detectors/squarespace/squarespace_test.go+++ b/pkg/detectors/squarespace/squarespace_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package squarespaceimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSquarespace_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SQUARESPACE_TOKEN")- inactiveSecret := testSecrets.MustGetField("SQUARESPACE_INACTIVE")+var (+ validPattern = "9308f78b-8315-a8a6-9c9c-a4b41488508f"+ invalidPattern = "9308f78b?8315-a8a6-9c9c-a4b41488508f"+ keyword = "squarespace"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSquarespace_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a squarespace secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Squarespace,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword squarespace",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a squarespace secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Squarespace,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Squarespace.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Squarespace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/squareup/squareup_integration_test.go b/pkg/detectors/squareup/squareup_integration_test.gonew file mode 100644index 000000000..fce1cce50--- /dev/null+++ b/pkg/detectors/squareup/squareup_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package squareup++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSquareup_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SQUAREUP")+ inactiveSecret := testSecrets.MustGetField("SQUAREUP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squareup secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Squareup,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a squareup secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Squareup,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Squareup.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Squareup.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/squareup/squareup_test.go b/pkg/detectors/squareup/squareup_test.goindex fce1cce50..787eee9c9 100644--- a/pkg/detectors/squareup/squareup_test.go+++ b/pkg/detectors/squareup/squareup_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package squareupimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSquareup_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SQUAREUP")- inactiveSecret := testSecrets.MustGetField("SQUAREUP_INACTIVE")+var (+ validPattern = "sq0idp-CIlelDA4a0mIksYXPGwUzy"+ invalidPattern = "sq0idp-CIlelDA?a0mIksYXPGwUzy"+ keyword = "squareup"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSquareup_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a squareup secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Squareup,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a squareup secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Squareup,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword squareup",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Squareup.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Squareup.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sslmate/sslmate_integration_test.go b/pkg/detectors/sslmate/sslmate_integration_test.gonew file mode 100644index 000000000..7c5f39660--- /dev/null+++ b/pkg/detectors/sslmate/sslmate_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package sslmate++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSslMate_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SSLMATE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SSLMATE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sslmate secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SslMate,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sslmate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SslMate,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SslMate.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SslMate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/sslmate/sslmate_test.go b/pkg/detectors/sslmate/sslmate_test.goindex 8244136b1..ee9cf6e14 100644--- a/pkg/detectors/sslmate/sslmate_test.go+++ b/pkg/detectors/sslmate/sslmate_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package sslmateimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSslMate_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SSLMATE_TOKEN")- inactiveSecret := testSecrets.MustGetField("SSLMATE_INACTIVE")+var (+ validPattern = "4RCKoI9YgcI43rpDbhyYpGdHtNaG1DLcwEYR"+ invalidPattern = "4RCKoI9YgcI43rpDbh?YpGdHtNaG1DLcwEYR"+ keyword = "sslmate"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSslMate_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sslmate secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SslMate,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sslmate",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sslmate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SslMate,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SslMate.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SslMate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/statuscake/statuscake_integration_test.go b/pkg/detectors/statuscake/statuscake_integration_test.gonew file mode 100644index 000000000..8cd8609cf--- /dev/null+++ b/pkg/detectors/statuscake/statuscake_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package statuscake++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStatuscake_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STATUSCAKE")+ inactiveSecret := testSecrets.MustGetField("STATUSCAKE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a statuscake secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Statuscake,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a statuscake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Statuscake,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Statuscake.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Statuscake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/statuscake/statuscake_test.go b/pkg/detectors/statuscake/statuscake_test.goindex 291347382..041ae4efe 100644--- a/pkg/detectors/statuscake/statuscake_test.go+++ b/pkg/detectors/statuscake/statuscake_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package statuscakeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStatuscake_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STATUSCAKE")- inactiveSecret := testSecrets.MustGetField("STATUSCAKE_INACTIVE")+var (+ validPattern = "jwI4HrRAbQV4rUYYD76C"+ invalidPattern = "jwI4HrRAbQ?4rUYYD76C"+ keyword = "statuscake"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStatuscake_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a statuscake secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Statuscake,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword statuscake",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a statuscake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Statuscake,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Statuscake.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Statuscake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/statuspage/statuspage_integration_test.go b/pkg/detectors/statuspage/statuspage_integration_test.gonew file mode 100644index 000000000..c2c8335dc--- /dev/null+++ b/pkg/detectors/statuspage/statuspage_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package statuspage++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStatuspage_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STATUSPAGE")+ inactiveSecret := testSecrets.MustGetField("STATUSPAGE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a statuspage secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Statuspage,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a statuspage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Statuspage,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Statuspage.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Statuspage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/statuspage/statuspage_test.go b/pkg/detectors/statuspage/statuspage_test.goindex f66e0a71d..197ea035d 100644--- a/pkg/detectors/statuspage/statuspage_test.go+++ b/pkg/detectors/statuspage/statuspage_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package statuspageimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStatuspage_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STATUSPAGE")- inactiveSecret := testSecrets.MustGetField("STATUSPAGE_INACTIVE")+var (+ validPattern = "0c8xiakjyy42sep7fs17pzq08ra0e4aj04uq"+ invalidPattern = "0c8xiakjyy42sep7fs?7pzq08ra0e4aj04uq"+ keyword = "statuspage"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStatuspage_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a statuspage secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Statuspage,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword statuspage",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a statuspage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Statuspage,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Statuspage.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Statuspage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/statuspal/statuspal_integration_test.go b/pkg/detectors/statuspal/statuspal_integration_test.gonew file mode 100644index 000000000..2f1a9547b--- /dev/null+++ b/pkg/detectors/statuspal/statuspal_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package statuspal++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStatuspal_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STATUSPAL")+ inactiveSecret := testSecrets.MustGetField("STATUSPAL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a statuspal secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Statuspal,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a statuspal secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Statuspal,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Statuspal.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Statuspal.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/statuspal/statuspal_test.go b/pkg/detectors/statuspal/statuspal_test.goindex 8f43e6141..10fcfee9a 100644--- a/pkg/detectors/statuspal/statuspal_test.go+++ b/pkg/detectors/statuspal/statuspal_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package statuspalimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStatuspal_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STATUSPAL")- inactiveSecret := testSecrets.MustGetField("STATUSPAL_INACTIVE")+var (+ validPattern = "FIYuLMuzF8XPqwPQzEiQFzJziMhcrAfH"+ invalidPattern = "FIYuLMu?F8XPqwPQzEiQFzJziMhcrAfH"+ keyword = "statuspal"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStatuspal_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a statuspal secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Statuspal,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword statuspal",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a statuspal secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Statuspal,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Statuspal.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Statuspal.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/stitchdata/stitchdata_integration_test.go b/pkg/detectors/stitchdata/stitchdata_integration_test.gonew file mode 100644index 000000000..e3f0c9a3f--- /dev/null+++ b/pkg/detectors/stitchdata/stitchdata_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package stitchdata++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStitchdata_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STITCHDATA")+ inactiveSecret := testSecrets.MustGetField("STITCHDATA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stitchdata,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stitchdata,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stitchdata.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stitchdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stitchdata/stitchdata_test.go b/pkg/detectors/stitchdata/stitchdata_test.goindex 42120ff03..fc3d40cd0 100644--- a/pkg/detectors/stitchdata/stitchdata_test.go+++ b/pkg/detectors/stitchdata/stitchdata_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package stitchdataimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStitchdata_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STITCHDATA")- inactiveSecret := testSecrets.MustGetField("STITCHDATA_INACTIVE")+var (+ validPattern = "8n4bfv0w_ygprt3xkr3gz1ez6jmz0oo9eva"+ invalidPattern = "8n4bfv0w_ygp?t3xkr3gz1ez6jmz0oo9eva"+ keyword = "stitchdata"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStitchdata_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stitchdata,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword stitchdata",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stitchdata,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stitchdata.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Stitchdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/stockdata/stockdata_integration_test.go b/pkg/detectors/stockdata/stockdata_integration_test.gonew file mode 100644index 000000000..d7794858e--- /dev/null+++ b/pkg/detectors/stockdata/stockdata_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package stockdata++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStockdata_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STOCKDATA")+ inactiveSecret := testSecrets.MustGetField("STOCKDATA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stockdata secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stockdata,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stockdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stockdata,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stockdata.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stockdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stockdata/stockdata_test.go b/pkg/detectors/stockdata/stockdata_test.goindex dd4654864..04bb01fef 100644--- a/pkg/detectors/stockdata/stockdata_test.go+++ b/pkg/detectors/stockdata/stockdata_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package stockdataimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStockdata_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STOCKDATA")- inactiveSecret := testSecrets.MustGetField("STOCKDATA_INACTIVE")+var (+ validPattern = "94ny9HMpy5rGR7jI3Z2fjBQpuU8gQofhlw4u3sRr"+ invalidPattern = "94ny9HMpy5rGR7?I3Z2fjBQpuU8gQofhlw4u3sRr"+ keyword = "stockdata"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStockdata_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stockdata secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stockdata,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword stockdata",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stockdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stockdata,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stockdata.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Stockdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/storecove/storecove_integration_test.go b/pkg/detectors/storecove/storecove_integration_test.gonew file mode 100644index 000000000..5adfc8b7e--- /dev/null+++ b/pkg/detectors/storecove/storecove_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package storecove++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStorecove_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STORECOVE")+ inactiveSecret := testSecrets.MustGetField("STORECOVE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a storecove secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Storecove,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a storecove secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Storecove,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Storecove.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Storecove.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/storecove/storecove_test.go b/pkg/detectors/storecove/storecove_test.goindex 42013daad..7fbabecd9 100644--- a/pkg/detectors/storecove/storecove_test.go+++ b/pkg/detectors/storecove/storecove_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package storecoveimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStorecove_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STORECOVE")- inactiveSecret := testSecrets.MustGetField("STORECOVE_INACTIVE")+var (+ validPattern = "yDDqj4B00b1m_HymYpqVRbPHWi7njN60NYsT0tpHvEe"+ invalidPattern = "yDDqj4B00?1m_HymYpqVRbPHWi7njN60NYsT0tpHvEe"+ keyword = "storecove"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStorecove_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a storecove secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Storecove,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword storecove",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a storecove secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Storecove,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Storecove.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Storecove.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/stormboard/stormboard_integration_test.go b/pkg/detectors/stormboard/stormboard_integration_test.gonew file mode 100644index 000000000..cf537ef23--- /dev/null+++ b/pkg/detectors/stormboard/stormboard_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package stormboard++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStormboard_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STORMBOARD")+ inactiveSecret := testSecrets.MustGetField("STORMBOARD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stormboard secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stormboard,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stormboard secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stormboard,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stormboard.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stormboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stormboard/stormboard_test.go b/pkg/detectors/stormboard/stormboard_test.goindex 2f1618fa1..ecc3d0fc6 100644--- a/pkg/detectors/stormboard/stormboard_test.go+++ b/pkg/detectors/stormboard/stormboard_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package stormboardimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStormboard_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STORMBOARD")- inactiveSecret := testSecrets.MustGetField("STORMBOARD_INACTIVE")+var (+ validPattern = "46e7fa9b370dd56209a5610630a45bb9a54fa1e9"+ invalidPattern = "4?e7fa9b370dd56209a5610630a45bb9a54fa1e9"+ keyword = "stormboard"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStormboard_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stormboard secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stormboard,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword stormboard",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stormboard secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stormboard,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stormboard.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Stormboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/stormglass/stormglass_integration_test.go b/pkg/detectors/stormglass/stormglass_integration_test.gonew file mode 100644index 000000000..8bdb9dfb8--- /dev/null+++ b/pkg/detectors/stormglass/stormglass_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package stormglass++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStormglass_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STORMGLASS")+ inactiveSecret := testSecrets.MustGetField("STORMGLASS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stormglass secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stormglass,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stormglass secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stormglass,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stormglass.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stormglass.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stormglass/stormglass_test.go b/pkg/detectors/stormglass/stormglass_test.goindex d4474a05f..26198eee3 100644--- a/pkg/detectors/stormglass/stormglass_test.go+++ b/pkg/detectors/stormglass/stormglass_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package stormglassimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStormglass_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STORMGLASS")- inactiveSecret := testSecrets.MustGetField("STORMGLASS_INACTIVE")+var (+ validPattern = "1sijzovx0w8ru40qteilgbxej6a5A1moc7s9b1p3mx9472dho5svv5yA7b0aqzu-Ai-z-r1xz"+ invalidPattern = "1sijzovx0w8ru40qteilg?xej6a5A1moc7s9b1p3mx9472dho5svv5yA7b0aqzu-Ai-z-r1xz"+ keyword = "stormglass"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStormglass_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stormglass secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stormglass,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword stormglass",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stormglass secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stormglass,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stormglass.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Stormglass.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/storyblok/storyblok_integration_test.go b/pkg/detectors/storyblok/storyblok_integration_test.gonew file mode 100644index 000000000..0320129eb--- /dev/null+++ b/pkg/detectors/storyblok/storyblok_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package storyblok++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStoryblok_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STORYBLOK_TOKEN")+ inactiveSecret := testSecrets.MustGetField("STORYBLOK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a storyblok secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Storyblok,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a storyblok secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Storyblok,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Storyblok.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Storyblok.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/storyblok/storyblok_test.go b/pkg/detectors/storyblok/storyblok_test.goindex 85c41035a..3afcca6b3 100644--- a/pkg/detectors/storyblok/storyblok_test.go+++ b/pkg/detectors/storyblok/storyblok_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package storyblokimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStoryblok_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STORYBLOK_TOKEN")- inactiveSecret := testSecrets.MustGetField("STORYBLOK_INACTIVE")+var (+ validPattern = "ZXIic758xXghN5SOkRI57ytt"+ invalidPattern = "ZXIic758xXgh?5SOkRI57ytt"+ keyword = "storyblok"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStoryblok_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a storyblok secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Storyblok,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword storyblok",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a storyblok secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Storyblok,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Storyblok.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Storyblok.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/storychief/storychief_integration_test.go b/pkg/detectors/storychief/storychief_integration_test.gonew file mode 100644index 000000000..dade17d2a--- /dev/null+++ b/pkg/detectors/storychief/storychief_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package storychief++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStorychief_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STORYCHIEF")+ inactiveSecret := testSecrets.MustGetField("STORYCHIEF_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a storychief secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Storychief,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a storychief secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Storychief,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Storychief.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Storychief.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/storychief/storychief_test.go b/pkg/detectors/storychief/storychief_test.goindex dade17d2a..a03e5c203 100644--- a/pkg/detectors/storychief/storychief_test.go+++ b/pkg/detectors/storychief/storychief_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package storychiefimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStorychief_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STORYCHIEF")- inactiveSecret := testSecrets.MustGetField("STORYCHIEF_INACTIVE")+var (+ validPattern = "OjhRmEboKs5kYVWJOrpTrloSRsAdkEl8PK-eHrSID5TRR.59-m5ezCY1dcJqJCxbV.2.LeGuqnhuouYkaiJ-iq90X-XX0J3IcT-ReLn.lAU0FfkvHOoOFp4k8w0-nDKAI8irzT5.pi7bmathhUdZO40-Rb59B6M0h40LbAkcvW49YP_-xXqGV_s.tCRbbzeUWt.Y9cFzfrQfRaVlTTqF5AC4mTEF4UxCSHK7uEE3OdzKhRehslviyKozUqer9HEZQ941rCqeEpt8kDcC0GOZFskrqu-EynCRJRhK6cv682e3HoRqE9x5FtcJ3gPfJEA70yHF5gTT0gh1KYDPKoJ-SQ7cjEBxxn-NbLu_WD3HW8DsjVDu1MgqrjqrnFNQvEAqYM7RI.RurVC38TP-5PioJqzJbvpNzkvwFjDrPpHdcivLgDEJUXS39PkIZPb1WCk.LPMOj5MBAwgn7TuADlV34Tael2FxygTPzA6ZVHBzm4aoUZ94Fwcm1vAkKJPeydsu33lmJ73e55pp7IhFyRO6MdPgLHqm3XkUhleU4yDUAEPWMyhilzOEO1t3nP3plfPjZU1.A1VEgWoOjvhs61qAMj2O6YsVFc7nU-PRlOpqj7yJNRmLWnJGVZW1UQLYwo5urJTb92u8BBPe179Eldzk5-xQ994NnrROnAK5DXkwdw9KII85fVBof8LGei1ocVEzYodTVvVY75iXaVmIP3Sf3dqkumW9Jdik-G-Lz.tvyJMTUSmtbX1oXaByyInng89h0Ah1O7D36nUm-gSOOgjJAssWCF0jiOSb2ps7BCdArjd5BVEOhewpodrtDs.iOncs8dtMSmyA5N7Jhzo2eINenb9dhJ7yQmskzhQcN-jpHKLpiL.w4lqCZ5X.uI_oDjx6V_7bJQK07uWCEB8xiwTRCCnB0mZYmi5q0WpG4sCY2xIW"+ invalidPattern = "O?hRmEboKs5kYVWJOrpTrloSRsAdkEl8PK-eHrSID5TRR.59-m5ezCY1dcJqJCxbV.2.LeGuqnhuouYkaiJ-iq90X-XX0J3IcT-ReLn.lAU0FfkvHOoOFp4k8w0-nDKAI8irzT5.pi7bmathhUdZO40-Rb59B6M0h40LbAkcvW49YP_-xXqGV_s.tCRbbzeUWt.Y9cFzfrQfRaVlTTqF5AC4mTEF4UxCSHK7uEE3OdzKhRehslviyKozUqer9HEZQ941rCqeEpt8kDcC0GOZFskrqu-EynCRJRhK6cv682e3HoRqE9x5FtcJ3gPfJEA70yHF5gTT0gh1KYDPKoJ-SQ7cjEBxxn-NbLu_WD3HW8DsjVDu1MgqrjqrnFNQvEAqYM7RI.RurVC38TP-5PioJqzJbvpNzkvwFjDrPpHdcivLgDEJUXS39PkIZPb1WCk.LPMOj5MBAwgn7TuADlV34Tael2FxygTPzA6ZVHBzm4aoUZ94Fwcm1vAkKJPeydsu33lmJ73e55pp7IhFyRO6MdPgLHqm3XkUhleU4yDUAEPWMyhilzOEO1t3nP3plfPjZU1.A1VEgWoOjvhs61qAMj2O6YsVFc7nU-PRlOpqj7yJNRmLWnJGVZW1UQLYwo5urJTb92u8BBPe179Eldzk5-xQ994NnrROnAK5DXkwdw9KII85fVBof8LGei1ocVEzYodTVvVY75iXaVmIP3Sf3dqkumW9Jdik-G-Lz.tvyJMTUSmtbX1oXaByyInng89h0Ah1O7D36nUm-gSOOgjJAssWCF0jiOSb2ps7BCdArjd5BVEOhewpodrtDs.iOncs8dtMSmyA5N7Jhzo2eINenb9dhJ7yQmskzhQcN-jpHKLpiL.w4lqCZ5X.uI_oDjx6V_7bJQK07uWCEB8xiwTRCCnB0mZYmi5q0WpG4sCY2xIW"+ keyword = "storychief"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStorychief_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a storychief secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Storychief,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword storychief",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a storychief secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Storychief,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Storychief.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Storychief.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/strava/strava_integration_test.go b/pkg/detectors/strava/strava_integration_test.gonew file mode 100644index 000000000..908965540--- /dev/null+++ b/pkg/detectors/strava/strava_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package strava++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStrava_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("STRAVA_ID")+ secret := testSecrets.MustGetField("STRAVA_SECRET")+ token := testSecrets.MustGetField("STRAVA_TOKEN")+ inactiveSecret := testSecrets.MustGetField("STRAVA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s", secret, id, token)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Strava,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s but not valid", inactiveSecret, id, token)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Strava,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Strava.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Strava.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/strava/strava_test.go b/pkg/detectors/strava/strava_test.goindex 93b10ddea..204f91382 100644--- a/pkg/detectors/strava/strava_test.go+++ b/pkg/detectors/strava/strava_test.go@@ -1,122 +1,85 @@-//go:build detectors-// +build detectors-package stravaimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStrava_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("STRAVA_ID")- secret := testSecrets.MustGetField("STRAVA_SECRET")- token := testSecrets.MustGetField("STRAVA_TOKEN")- inactiveSecret := testSecrets.MustGetField("STRAVA_INACTIVE")+var (+ validId = "57732"+ invalidId = "5?732"+ validSecret = "ateo889bcja9fyt4gff2kqe187uhy8l0il4vf2cw"+ invalidSecret = "ateo889bcja9fyt4gff2?qe187uhy8l0il4vf2cw"+ validKey = "e8qmzsdc0qbq4hsal5m4q1pz5s2yloydbl1bjc1u"+ invalidKey = "e8?mzsdc0qbq4hsal5m4q1pz5s2yloydbl1bjc1u"+ keyword = "strava"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStrava_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s", secret, id, token)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Strava,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword strava",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validSecret, keyword, validKey),+ want: []string{validId + validSecret, validId + validKey, validId + validKey, validId + validSecret},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s but not valid", inactiveSecret, id, token)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Strava,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidSecret, keyword, invalidKey),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Strava.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Strava.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/streak/streak_integration_test.go b/pkg/detectors/streak/streak_integration_test.gonew file mode 100644index 000000000..38c4e7d45--- /dev/null+++ b/pkg/detectors/streak/streak_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package streak++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStreak_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STREAK")+ inactiveSecret := testSecrets.MustGetField("STREAK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a streak secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Streak,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a streak secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Streak,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Streak.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Streak.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/streak/streak_test.go b/pkg/detectors/streak/streak_test.goindex a9d256ae0..ea1ffeff4 100644--- a/pkg/detectors/streak/streak_test.go+++ b/pkg/detectors/streak/streak_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package streakimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStreak_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STREAK")- inactiveSecret := testSecrets.MustGetField("STREAK_INACTIVE")+var (+ validPattern = "bf1949fA2574fb6b2deb94AdA51f0719"+ invalidPattern = "bf1949fA?574fb6b2deb94AdA51f0719"+ keyword = "streak"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStreak_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a streak secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Streak,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword streak",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a streak secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Streak,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Streak.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Streak.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/stripe/stripe_integration_test.go b/pkg/detectors/stripe/stripe_integration_test.gonew file mode 100644index 000000000..19bb0836a--- /dev/null+++ b/pkg/detectors/stripe/stripe_integration_test.go@@ -0,0 +1,126 @@+//go:build detectors+// +build detectors++package stripe++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStripe_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STRIPE_SECRET")+ secretInactive := testSecrets.MustGetField("STRIPE_INACTIVE")+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stripe,+ Verified: true,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stripe,+ Verified: false,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stripe.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ got[i].AnalysisInfo = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stripe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stripe/stripe_test.go b/pkg/detectors/stripe/stripe_test.goindex 905a0d721..5c6c9a046 100644--- a/pkg/detectors/stripe/stripe_test.go+++ b/pkg/detectors/stripe/stripe_test.go@@ -1,125 +1,81 @@-//go:build detectors-// +build detectors-package stripeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStripe_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STRIPE_SECRET")- secretInactive := testSecrets.MustGetField("STRIPE_INACTIVE")- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validPattern = "rk_live_HUOlkIKhNEYOPS0oDSwGwJHbg4xGaXNeJZ2CdvDGeVZQHljoq5TuFwQHgME3W"+ invalidPattern = "?k_live_HUOlkIKhNEYOPS0oDSwGwJHbg4xGaXNeJZ2CdvDGeVZQHljoq5TuFwQHgME3?"+ keyword = "stripe"+)++func TestStripe_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stripe,- Verified: true,- ExtraData: map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",- },- },- },- wantErr: false,+ name: "valid pattern - with keyword stripe",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stripe,- Verified: false,- ExtraData: map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",- },- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stripe.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil- got[i].AnalysisInfo = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Stripe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/stripo/stripo_integration_test.go b/pkg/detectors/stripo/stripo_integration_test.gonew file mode 100644index 000000000..ae2bb3c72--- /dev/null+++ b/pkg/detectors/stripo/stripo_integration_test.go@@ -0,0 +1,127 @@+//go:build detectors+// +build detectors++package stripo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStripo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STRIPO")+ inactiveSecret := testSecrets.MustGetField("STRIPO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stripo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stripo,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stripo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stripo,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stripo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Stripo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stripo/stripo_test.go b/pkg/detectors/stripo/stripo_test.goindex 3c6671873..fd66f01bd 100644--- a/pkg/detectors/stripo/stripo_test.go+++ b/pkg/detectors/stripo/stripo_test.go@@ -1,127 +1,90 @@-//go:build detectors-// +build detectors-package stripoimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStripo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STRIPO")- inactiveSecret := testSecrets.MustGetField("STRIPO_INACTIVE")+var (+ validPattern = "eyJhbGciOiJIUzI1NiJ9.qflXV62P6Y51gAFD9pnq9FpcgDQOfr68GtBUARUvk7gzbAfg6eF66Txb4R9idT41JsG5CqTKXMfMnzpPVByJRFjtfXIFasywYCd53PqB774bhbBFgfEznjdQjOu8ui910q.8QQbrVsL16ya98sn8yyjLbBX21bix2G_QhFg_rKN9Bv"+ invalidPattern = "ayJhbGciOiJIUzI1NiJ9.qflXV62P6Y51gAFD9pnq9FpcgDQOfr68GtBUARUvk7gzbAfg6eF66Txb4R9idT41JsG5CqTKXMfMnzpPVByJRFjtfXIFasywYCd53PqB774bhbBFgfEznjdQjOu8ui910q.8QQbrVsL16ya98sn8yyjLbBX21bix2G_QhFg_rKN9Bv"+ keyword = "stripo"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStripo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stripo secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stripo,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword stripo",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stripo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stripo,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stripo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Stripo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/stytch/stytch_integration_test.go b/pkg/detectors/stytch/stytch_integration_test.gonew file mode 100644index 000000000..6de25ac8a--- /dev/null+++ b/pkg/detectors/stytch/stytch_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package stytch++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestStytch_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("STYTCH")+ id := testSecrets.MustGetField("STYTCH_ID")+ inactiveSecret := testSecrets.MustGetField("STYTCH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stytch,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Stytch,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Stytch.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Stytch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/stytch/stytch_test.go b/pkg/detectors/stytch/stytch_test.goindex 0d1722dae..5a6d09c3e 100644--- a/pkg/detectors/stytch/stytch_test.go+++ b/pkg/detectors/stytch/stytch_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package stytchimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestStytch_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("STYTCH")- id := testSecrets.MustGetField("STYTCH_ID")- inactiveSecret := testSecrets.MustGetField("STYTCH_INACTIVE")+var (+ validKey = "TSIDpkOi1MKTudKSFs-KXGdOeH3ECTeXWtX7OAr4Y9k31m8="+ invalidKey = "TSIDpkOi1MKTudKSFs-KXGdO?H3ECTeXWtX7OAr4Y9k31m8="+ validId = "k7hi7kqgb6a8rqi0b8nbna1p-o32rv9xzlnvzbd5oa9yr-vtp"+ invalidId = "k7hi7kqgb6a8rqi0b8nbna1p?o32rv9xzlnvzbd5oa9yr-vtp"+ keyword = "stytch"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestStytch_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stytch,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword stytch",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Stytch,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Stytch.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Stytch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/sugester/sugester_integration_test.go b/pkg/detectors/sugester/sugester_integration_test.gonew file mode 100644index 000000000..4b27ef9a3--- /dev/null+++ b/pkg/detectors/sugester/sugester_integration_test.go@@ -0,0 +1,118 @@+//go:build detectors+// +build detectors++package sugester++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSugester_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SUGESTER")+ domain := testSecrets.MustGetField("SUGESTER_DOMAIN")+ inactiveSecret := testSecrets.MustGetField("SUGESTER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s", secret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sugester,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Sugester,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Sugester.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Sugester.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/sugester/sugester_test.go b/pkg/detectors/sugester/sugester_test.goindex 4b27ef9a3..aa9d07dfa 100644--- a/pkg/detectors/sugester/sugester_test.go+++ b/pkg/detectors/sugester/sugester_test.go@@ -1,117 +1,82 @@-//go:build detectors-// +build detectors-package sugesterimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSugester_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SUGESTER")- domain := testSecrets.MustGetField("SUGESTER_DOMAIN")- inactiveSecret := testSecrets.MustGetField("SUGESTER_INACTIVE")+var (+ validKey = "RfQEMjlVIolNGUebkCS7iw0MevjgAD06"+ invalidKey = "RfQEMjlVIolNGUeb?CS7iw0MevjgAD06"+ validDomain = "h$6o0+jTX"+ invalidDomain = "?$6o0+jT?"+ keyword = "sugester"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSugester_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s", secret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sugester,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword sugester",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),+ want: []string{validKey, validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Sugester,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Sugester.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Sugester.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/sumologickey/sumologickey_integration_test.go b/pkg/detectors/sumologickey/sumologickey_integration_test.goindex e95e34041..4776f5b49 100644--- a/pkg/detectors/sumologickey/sumologickey_integration_test.go+++ b/pkg/detectors/sumologickey/sumologickey_integration_test.go@@ -10,9 +10,9 @@ import ("time""github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")diff --git a/pkg/detectors/supabasetoken/supabasetoken_integration_test.go b/pkg/detectors/supabasetoken/supabasetoken_integration_test.gonew file mode 100644index 000000000..5c97a18ad--- /dev/null+++ b/pkg/detectors/supabasetoken/supabasetoken_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package supabasetoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSupabasetoken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SUPABASETOKEN")+ inactiveSecret := testSecrets.MustGetField("SUPABASETOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SupabaseToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SupabaseToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Supabasetoken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Supabasetoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/supabasetoken/supabasetoken_test.go b/pkg/detectors/supabasetoken/supabasetoken_test.goindex 840e235b8..dcdb9a406 100644--- a/pkg/detectors/supabasetoken/supabasetoken_test.go+++ b/pkg/detectors/supabasetoken/supabasetoken_test.go@@ -1,121 +1,81 @@-//go:build detectors-// +build detectors-package supabasetokenimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSupabasetoken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SUPABASETOKEN")- inactiveSecret := testSecrets.MustGetField("SUPABASETOKEN_INACTIVE")+var (+ validPattern = "sbp_6corwd8ltv0m2hx5av0bqswrldi43hk8qdtnxcs7"+ invalidPattern = "sbp_6corwd8ltv?m2hx5av0bqswrldi43hk8qdtnxcs7"+ keyword = "supabasetoken"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSupabasetoken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SupabaseToken,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword supabasetoken",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SupabaseToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Supabasetoken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Supabasetoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/supernotesapi/supernotesapi_integration_test.go b/pkg/detectors/supernotesapi/supernotesapi_integration_test.gonew file mode 100644index 000000000..cf6a075a1--- /dev/null+++ b/pkg/detectors/supernotesapi/supernotesapi_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package supernotesapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSuperNotesAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SUPERNOTESAPI")+ inactiveSecret := testSecrets.MustGetField("SUPERNOTESAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a supernotes secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SuperNotesAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a supernotes secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SuperNotesAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SuperNotesAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SuperNotesAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/supernotesapi/supernotesapi_test.go b/pkg/detectors/supernotesapi/supernotesapi_test.goindex cf6a075a1..43e926d9a 100644--- a/pkg/detectors/supernotesapi/supernotesapi_test.go+++ b/pkg/detectors/supernotesapi/supernotesapi_test.go@@ -1,116 +1,90 @@-//go:build detectors-// +build detectors-package supernotesapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSuperNotesAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SUPERNOTESAPI")- inactiveSecret := testSecrets.MustGetField("SUPERNOTESAPI_INACTIVE")+var (+ validPattern = "V77f640DIDmrPcYFMwCKPhAR7CkXDwXjmMGpzHVm6yC"+ invalidPattern = "V77f640DIDmrPcYFMwCKPh?R7CkXDwXjmMGpzHVm6yC"+ keyword = "supernotesapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSuperNotesAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a supernotes secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SuperNotesAPI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword supernotesapi",+ input: fmt.Sprintf("%s token = ' %s '", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a supernotes secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SuperNotesAPI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = ' %s ' | ' %s '", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = ' %s '", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = ' %s '", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SuperNotesAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SuperNotesAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/surveyanyplace/surveyanyplace_integration_test.go b/pkg/detectors/surveyanyplace/surveyanyplace_integration_test.gonew file mode 100644index 000000000..6a849f3e8--- /dev/null+++ b/pkg/detectors/surveyanyplace/surveyanyplace_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package surveyanyplace++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSurveyAnyplace_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SURVEYANYPLACE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SURVEYANYPLACE_INACTIVE")+ id := testSecrets.MustGetField("SURVEYANYPLACE_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SurveyAnyplace,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SurveyAnyplace,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SurveyAnyplace.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SurveyAnyplace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/surveyanyplace/surveyanyplace_test.go b/pkg/detectors/surveyanyplace/surveyanyplace_test.goindex 4f57a6215..538ec420d 100644--- a/pkg/detectors/surveyanyplace/surveyanyplace_test.go+++ b/pkg/detectors/surveyanyplace/surveyanyplace_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package surveyanyplaceimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSurveyAnyplace_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SURVEYANYPLACE_TOKEN")- inactiveSecret := testSecrets.MustGetField("SURVEYANYPLACE_INACTIVE")- id := testSecrets.MustGetField("SURVEYANYPLACE_ID")+var (+ validKey = "zDbWB2TozuNJj8o56FIB0PVfE7LfjHFd"+ invalidKey = "zDbWB2TozuNJj8o5?FIB0PVfE7LfjHFd"+ validId = "UEN9VEUk4z1oY-pV2CPLZBAnn0rPWY3LCqNv"+ invalidId = "UEN9VEUk4z1oY-pV2C?LZBAnn0rPWY3LCqNv"+ keyword = "surveyanyplace"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSurveyAnyplace_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SurveyAnyplace,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword surveyanyplace",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SurveyAnyplace,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SurveyAnyplace.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SurveyAnyplace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/surveybot/surveybot_integration_test.go b/pkg/detectors/surveybot/surveybot_integration_test.gonew file mode 100644index 000000000..dd567c2c1--- /dev/null+++ b/pkg/detectors/surveybot/surveybot_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package surveybot++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSurveyBot_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SURVEYBOT_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SURVEYBOT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a surveybot secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SurveyBot,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a surveybot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SurveyBot,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SurveyBot.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SurveyBot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/surveybot/surveybot_test.go b/pkg/detectors/surveybot/surveybot_test.goindex 9fe32246e..c8210aca3 100644--- a/pkg/detectors/surveybot/surveybot_test.go+++ b/pkg/detectors/surveybot/surveybot_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package surveybotimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSurveyBot_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SURVEYBOT_TOKEN")- inactiveSecret := testSecrets.MustGetField("SURVEYBOT_INACTIVE")+var (+ validPattern = "ZGPxFPcng3xOumtiY14y8Gtp8IAp5pHlnxOf28l3ThncGCOHdU4o01muxq4UqAJP28s4hfkASbzVY0Qi"+ invalidPattern = "ZGPxFPcng3xOumtiY14y8Gtp8IAp5pHlnxOf28l3?hncGCOHdU4o01muxq4UqAJP28s4hfkASbzVY0Qi"+ keyword = "surveybot"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSurveyBot_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a surveybot secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SurveyBot,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword surveybot",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a surveybot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SurveyBot,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SurveyBot.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SurveyBot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/surveysparrow/surveysparrow_integration_test.go b/pkg/detectors/surveysparrow/surveysparrow_integration_test.gonew file mode 100644index 000000000..607b5a53f--- /dev/null+++ b/pkg/detectors/surveysparrow/surveysparrow_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package surveysparrow++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSurveySparrow_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SURVEYSPARROW_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SURVEYSPARROW_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SurveySparrow,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_SurveySparrow,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("SurveySparrow.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("SurveySparrow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/surveysparrow/surveysparrow_test.go b/pkg/detectors/surveysparrow/surveysparrow_test.goindex 2a7c7be70..c82d88b03 100644--- a/pkg/detectors/surveysparrow/surveysparrow_test.go+++ b/pkg/detectors/surveysparrow/surveysparrow_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package surveysparrowimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSurveySparrow_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SURVEYSPARROW_TOKEN")- inactiveSecret := testSecrets.MustGetField("SURVEYSPARROW_INACTIVE")+var (+ validPattern = "nRTqAmYY7E8pi6CaSf4JGgvxreYC-DEI6Q9De1EYLilWDuSHGOugKnWYdIz7ifGlFDol4v2hsYH7C9lu5Z63XmwP"+ invalidPattern = "nRTqAmYY7E8pi6CaSf4JGgvxreYC-?EI6Q9De1EYLilWDuSHGOugKnWYdIz7ifGlFDol4v2hsYH7C9lu5Z63XmwP"+ keyword = "surveysparrow"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSurveySparrow_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SurveySparrow,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword surveysparrow",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_SurveySparrow,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("SurveySparrow.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("SurveySparrow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/survicate/survicate_integration_test.go b/pkg/detectors/survicate/survicate_integration_test.gonew file mode 100644index 000000000..61db64034--- /dev/null+++ b/pkg/detectors/survicate/survicate_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package survicate++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSurvicate_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SURVICATE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("SURVICATE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a survicate secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Survicate,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a survicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Survicate,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Survicate.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Survicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/survicate/survicate_test.go b/pkg/detectors/survicate/survicate_test.goindex c9c93ca73..f4e2c9370 100644--- a/pkg/detectors/survicate/survicate_test.go+++ b/pkg/detectors/survicate/survicate_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package survicateimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSurvicate_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SURVICATE_TOKEN")- inactiveSecret := testSecrets.MustGetField("SURVICATE_INACTIVE")+var (+ validPattern = "lztej917o59y8x82f311v8d684eb2t6k"+ invalidPattern = "lztej917o59y8x82?311v8d684eb2t6k"+ keyword = "survicate"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSurvicate_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a survicate secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Survicate,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword survicate",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a survicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Survicate,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Survicate.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Survicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/swell/swell_integration_test.go b/pkg/detectors/swell/swell_integration_test.gonew file mode 100644index 000000000..90edf0702--- /dev/null+++ b/pkg/detectors/swell/swell_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package swell++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSwell_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SWELL")+ id := testSecrets.MustGetField("SWELL_ID")+ inactiveSecret := testSecrets.MustGetField("SWELL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Swell,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Swell,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Swell.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Swell.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/swell/swell_test.go b/pkg/detectors/swell/swell_test.goindex 1994ea923..00a6ef92c 100644--- a/pkg/detectors/swell/swell_test.go+++ b/pkg/detectors/swell/swell_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package swellimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSwell_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SWELL")- id := testSecrets.MustGetField("SWELL_ID")- inactiveSecret := testSecrets.MustGetField("SWELL_INACTIVE")+var (+ validKey = "Mkff1bQx72mxhvNdCsudapMP7eery91G"+ invalidKey = "Mkff1bQx72mxhvNd?sudapMP7eery91G"+ validId = "0aOWuIMHYK6F30I1"+ invalidId = "?aOWuIMHYK6F30I?"+ keyword = "swell"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSwell_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Swell,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword swell",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Swell,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Swell.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Swell.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/swiftype/swiftype_integration_test.go b/pkg/detectors/swiftype/swiftype_integration_test.gonew file mode 100644index 000000000..bafbcb78a--- /dev/null+++ b/pkg/detectors/swiftype/swiftype_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package swiftype++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestSwiftype_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("SWIFTYPE")+ inactiveSecret := testSecrets.MustGetField("SWIFTYPE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a swiftype secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Swiftype,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a swiftype secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Swiftype,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Swiftype.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Swiftype.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/swiftype/swiftype_test.go b/pkg/detectors/swiftype/swiftype_test.goindex 95930c642..3715353e8 100644--- a/pkg/detectors/swiftype/swiftype_test.go+++ b/pkg/detectors/swiftype/swiftype_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package swiftypeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestSwiftype_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SWIFTYPE")- inactiveSecret := testSecrets.MustGetField("SWIFTYPE_INACTIVE")+var (+ validPattern = "hJgyi1_y8ohpS-N2vlXE"+ invalidPattern = "?Jgyi1_y8ohpS-N2vlXE"+ keyword = "swiftype"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestSwiftype_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a swiftype secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Swiftype,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword swiftype",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a swiftype secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Swiftype,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Swiftype.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Swiftype.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}