Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
added pattern test cases for all detectors starting with Alphabet `a` (#3539)
Kashif Khan8897ed2a66fa
diff --git a/pkg/detectors/allsports/allsports_integration_test.go b/pkg/detectors/allsports/allsports_integration_test.gonew file mode 100644index 000000000..0619c62f7--- /dev/null+++ b/pkg/detectors/allsports/allsports_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package allsports++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAllsports_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ALLSPORTS")+ inactiveSecret := testSecrets.MustGetField("ALLSPORTS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a allsports secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Allsports,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a allsports secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Allsports,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Allsports.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Allsports.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/allsports/allsports_test.go b/pkg/detectors/allsports/allsports_test.goindex edf6454fc..81414fd8f 100644--- a/pkg/detectors/allsports/allsports_test.go+++ b/pkg/detectors/allsports/allsports_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package allsportsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "d1f2e3c4b5a6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1s2t3u4v5w6x7y8z9a0b1ce"+ invalidPattern = "Ad1f2e3c4b5a6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1sRt3u4v5w6x7y8z9a0b1cE")-func TestAllsports_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ALLSPORTS")- inactiveSecret := testSecrets.MustGetField("ALLSPORTS_INACTIVE")+func TestAllSports_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a allsports secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Allsports,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("allsports: '%s'", validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a allsports secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Allsports,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("allsports keyword is not close to the real key in the data\n = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("allsports: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Allsports.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Allsports.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/amadeus/amadeus_integration_test.go b/pkg/detectors/amadeus/amadeus_integration_test.gonew file mode 100644index 000000000..0f87057ca--- /dev/null+++ b/pkg/detectors/amadeus/amadeus_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package amadeus++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAmadeus_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("AMADEUS")+ secret := testSecrets.MustGetField("AMADEUS_SECRET")+ inactiveSecret := testSecrets.MustGetField("AMADEUS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Amadeus,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Amadeus,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Amadeus.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Amadeus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/amadeus/amadeus_test.go b/pkg/detectors/amadeus/amadeus_test.goindex 0f87057ca..5437f5f06 100644--- a/pkg/detectors/amadeus/amadeus_test.go+++ b/pkg/detectors/amadeus/amadeus_test.go@@ -1,121 +1,86 @@-//go:build detectors-// +build detectors-package amadeusimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6 / A1b2C3d4E5f6G7ho"+ invalidPattern = "A1b2$3d4E5f6G7h8I9j(K1l2M3n4O5)6 / A1b2C3d4E^f6G7#8")-func TestAmadeus_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("AMADEUS")- secret := testSecrets.MustGetField("AMADEUS_SECRET")- inactiveSecret := testSecrets.MustGetField("AMADEUS_INACTIVE")+func TestAmadeus_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Amadeus,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("amadeus: '%s'", validPattern),+ want: []string{"A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6A1b2C3d4E5f6G7ho"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Amadeus,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("amadeus keyword is not close to the real key in the data\n = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("amadeus: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Amadeus.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Amadeus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/ambee/ambee_integration_test.go b/pkg/detectors/ambee/ambee_integration_test.gonew file mode 100644index 000000000..6ff21de9d--- /dev/null+++ b/pkg/detectors/ambee/ambee_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package ambee++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAmbee_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AMBEE")+ inactiveSecret := testSecrets.MustGetField("AMBEE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ambee secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ambee,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ambee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ambee,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Ambee.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Ambee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/ambee/ambee_test.go b/pkg/detectors/ambee/ambee_test.goindex 6ff21de9d..12dc72b45 100644--- a/pkg/detectors/ambee/ambee_test.go+++ b/pkg/detectors/ambee/ambee_test.go@@ -1,120 +1,86 @@-//go:build detectors-// +build detectors-package ambeeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4"+ invalidPattern = "X3b4c5d6e7f8a9b0c1d2e3f4a5b6c7Z8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4")-func TestAmbee_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AMBEE")- inactiveSecret := testSecrets.MustGetField("AMBEE_INACTIVE")+func TestAmbee_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ambee secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ambee,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("ambee: '%s'", validPattern),+ want: []string{"a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ambee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ambee,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("ambee keyword is not close to the real key in the data\n = '%s'", validPattern),+ want: nil,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("ambee: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Ambee.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Ambee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey.go b/pkg/detectors/amplitudeapikey/amplitudeapikey.goindex 7f5a3d6e3..6e041d760 100644--- a/pkg/detectors/amplitudeapikey/amplitudeapikey.go+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey.go@@ -2,16 +2,17 @@ package amplitudeapikeyimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -51,6 +52,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}resSecretMatch := strings.TrimSpace(secretMatch[1])+ // regex for both key and secret are same so the set of strings could possibly be same as well+ if resMatch == resSecretMatch {+ continue+ }+s1 := detectors.Result{DetectorType: detectorspb.DetectorType_AmplitudeApiKey,Raw: []byte(resMatch),diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.gonew file mode 100644index 000000000..9028ed7fb--- /dev/null+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go@@ -0,0 +1,124 @@+//go:build detectors+// +build detectors++package amplitudeapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAmplitudeApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ key := testSecrets.MustGetField("AMPLITUDEAPI_KEY")+ secret := testSecrets.MustGetField("AMPLITUDEAPI_SECRET")+ inactiveKey := testSecrets.MustGetField("AMPLITUDEAPI_KEY_INACTIVE")+ inactiveSecret := testSecrets.MustGetField("AMPLITUDEAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within", key, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AmplitudeApiKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),++ data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AmplitudeApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AmplitudeApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AmplitudeApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.goindex 9028ed7fb..e9fb9c351 100644--- a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go@@ -1,124 +1,90 @@-//go:build detectors-// +build detectors-package amplitudeapikeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAmplitudeApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }+var (+ validPattern = `+ amplitude key = 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d+ amplitude secret = 5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e+ `+ invalidPattern = `+ amplitude key = 1a2b3c4d5e6f7a8g9c0d1e2f3a4b5c6d+ amplitude secret = 5b3c4d5e5f7a8b9c0d1r2f2f3a4b4c6e+ `+)- key := testSecrets.MustGetField("AMPLITUDEAPI_KEY")- secret := testSecrets.MustGetField("AMPLITUDEAPI_SECRET")- inactiveKey := testSecrets.MustGetField("AMPLITUDEAPI_KEY_INACTIVE")- inactiveSecret := testSecrets.MustGetField("AMPLITUDEAPI_INACTIVE")+func TestAmplitudeAPIKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within", key, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AmplitudeApiKey,- Verified: true,- },+ name: "valid pattern",+ input: fmt.Sprintf("amplitude: '%s'", validPattern),+ want: []string{+ "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e",+ "5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d",},- wantErr: false,},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),-- data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AmplitudeApiKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("amplitude: '%s'", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AmplitudeApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AmplitudeApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/anthropic/anthropic_integration_test.go b/pkg/detectors/anthropic/anthropic_integration_test.gonew file mode 100644index 000000000..64713de61--- /dev/null+++ b/pkg/detectors/anthropic/anthropic_integration_test.go@@ -0,0 +1,162 @@+//go:build detectors+// +build detectors++package anthropic++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAnthropic_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ANTHROPIC")+ inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Anthropic,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Anthropic,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Anthropic,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Anthropic,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/anthropic/anthropic_test.go b/pkg/detectors/anthropic/anthropic_test.goindex 64713de61..ba1f085ac 100644--- a/pkg/detectors/anthropic/anthropic_test.go+++ b/pkg/detectors/anthropic/anthropic_test.go@@ -1,161 +1,79 @@-//go:build detectors-// +build detectors-package anthropicimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "sk-ant-api03-abc123xyz-456def789ghij-klmnopqrstuvwx-3456yza789bcde-1234fghijklmnopby56aaaogaopaaaabc123xyzAA"+ invalidPattern = "sk-ant-api03-abc123xyz-456de-klMnopqrstuvwx-3456yza789bcde-1234fghijklmnopAA")-func TestAnthropic_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ANTHROPIC")- inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE")+func TestAnthropic_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Anthropic,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Anthropic,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Anthropic,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Anthropic,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/anypoint/anypoint.go b/pkg/detectors/anypoint/anypoint.goindex c12bcb5bc..d49643ceb 100644--- a/pkg/detectors/anypoint/anypoint.go+++ b/pkg/detectors/anypoint/anypoint.go@@ -3,10 +3,11 @@ package anypointimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -51,6 +52,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}orgRes := strings.TrimSpace(orgMatch[1])+ // regex for both key and org are same, so to avoid same string processing+ if resMatch == orgRes {+ continue+ }+s1 := detectors.Result{DetectorType: detectorspb.DetectorType_Anypoint,Raw: []byte(resMatch),diff --git a/pkg/detectors/anypoint/anypoint_integration_test.go b/pkg/detectors/anypoint/anypoint_integration_test.gonew file mode 100644index 000000000..e07331436--- /dev/null+++ b/pkg/detectors/anypoint/anypoint_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package anypoint++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAnypoint_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ANYPOINT_TOKEN")+ inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE")+ organizationId := testSecrets.MustGetField("ANYPOINT_ORGANIZATIONID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s", secret, organizationId)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Anypoint,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s but not valid", inactiveSecret, organizationId)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Anypoint,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Anypoint.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/anypoint/anypoint_test.go b/pkg/detectors/anypoint/anypoint_test.goindex e07331436..31f5a91df 100644--- a/pkg/detectors/anypoint/anypoint_test.go+++ b/pkg/detectors/anypoint/anypoint_test.go@@ -1,121 +1,107 @@-//go:build detectors-// +build detectors-package anypointimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = "1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6 / org: abcd1234-ef56-gh78-ij90-klmn1234opqr"+ complexPattern = `+ # Secret Configuration File+ # Organization details+ ORG_NAME=my_organization+ ORG_ID=abcd1234-ef56-gh78-ij90-klmn1234opqr++ # Database credentials+ DB_USERNAME=iamnotadmin+ DB_PASSWORD=8f3b6d3e7c9a2f5e++ # OAuth tokens+ ACCESS_TOKEN=abcxyz123+ REFRESH_TOKEN=zyxwvutsrqponmlkji9876543210abcd- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ # API keys+ API_KEY=sk-ant-api03-nothing-just-some-random-api-key-1234fghijklmnopAA+ SECRET_KEY=1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6++ # Endpoints+ SERVICE_URL=https://api.example.com/v1/resource+ `+ invalidPattern = "1a2b3C4d-5E6f-7g8H-9i0J-k1l2M3n4o5p6 / abcd1234-eF56-gH78-ij90-klmn1234opqr")-func TestAnypoint_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ANYPOINT_TOKEN")- inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE")- organizationId := testSecrets.MustGetField("ANYPOINT_ORGANIZATIONID")+func TestAnypoint_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s", secret, organizationId)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Anypoint,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("anypoint credentials: %s", validPattern),+ want: []string{"1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6abcd1234-ef56-gh78-ij90-klmn1234opqr"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s but not valid", inactiveSecret, organizationId)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Anypoint,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - complex",+ input: fmt.Sprintf("anypoint credentials: %s", complexPattern),+ want: []string{"1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6abcd1234-ef56-gh78-ij90-klmn1234opqr"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("anypoint credentials: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Anypoint.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apacta/apacta_integration_test.go b/pkg/detectors/apacta/apacta_integration_test.gonew file mode 100644index 000000000..d80e7f04b--- /dev/null+++ b/pkg/detectors/apacta/apacta_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package apacta++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApacta_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APACTA")+ inactiveSecret := testSecrets.MustGetField("APACTA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apacta secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apacta,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apacta secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apacta,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apacta.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apacta.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apacta/apacta_test.go b/pkg/detectors/apacta/apacta_test.goindex d80e7f04b..8b6fee13b 100644--- a/pkg/detectors/apacta/apacta_test.go+++ b/pkg/detectors/apacta/apacta_test.go@@ -1,120 +1,113 @@-//go:build detectors-// +build detectors-package apactaimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestApacta_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APACTA")- inactiveSecret := testSecrets.MustGetField("APACTA_INACTIVE")+var (+ validPattern = "abcd1234-ef56-gh78-ij90-klmn1234opqr"+ complexPattern = `+ func main() {+ url := "https://api.example.com/v1/resource"++ // Create a new request with the secret as a header+ req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte("{}")))+ if err != nil {+ fmt.Println("Error creating request:", err)+ return+ }++ apactaSecret := "Bearer abcd1234-ef56-gh78-ij90-klmn1234opqr"+ req.Header.Set("Authorization", apactaSecret)++ // Perform the request+ client := &http.Client{}+ resp, _ := client.Do(req)+ defer resp.Body.Close()- type args struct {- ctx context.Context- data []byte- verify bool+ // Check response status+ if resp.StatusCode == http.StatusOK {+ fmt.Println("Request successful!")+ } else {+ fmt.Println("Request failed with status:", resp.Status)+ }}+ `+ invalidPattern = "abcD$1234-ef56-gH78-ij90-klmn1234opqr"+)++func TestApacta_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apacta secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apacta,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("apacta credentials: %s", validPattern),+ want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apacta secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apacta,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - complex",+ input: complexPattern,+ want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("apacta credentials: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apacta.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apacta.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/api2cart/api2cart_integration_test.go b/pkg/detectors/api2cart/api2cart_integration_test.gonew file mode 100644index 000000000..f0243d52b--- /dev/null+++ b/pkg/detectors/api2cart/api2cart_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package api2cart++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApi2Cart_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("API2CART")+ inactiveSecret := testSecrets.MustGetField("API2CART_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a api2cart secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Api2Cart,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a api2cart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Api2Cart,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Api2Cart.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Api2Cart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/api2cart/api2cart_test.go b/pkg/detectors/api2cart/api2cart_test.goindex f0243d52b..c3fa39be9 100644--- a/pkg/detectors/api2cart/api2cart_test.go+++ b/pkg/detectors/api2cart/api2cart_test.go@@ -1,120 +1,93 @@-//go:build detectors-// +build detectors-package api2cartimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "0123456789abcdef0123456789abcdef"+ complexPattern = `+ To integrate with API2Cart, ensure you have the following credentials in your configuration file. Your API2CART key is 1234567890abcdef1234567890abcdef, which you will need to access the API securely.++ The following endpoints are available for your use:+ - Get Products: https://api.api2cart.com/v1.0/products/get+ - Add Product: https://api.api2cart.com/v1.0/products/add+ `+ invalidPattern = "0123456789Gbcde^0123456789abcdef")-func TestApi2Cart_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("API2CART")- inactiveSecret := testSecrets.MustGetField("API2CART_INACTIVE")+func TestApi2Cart_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a api2cart secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Api2Cart,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("api2cart credentials: %s", validPattern),+ want: []string{"0123456789abcdef0123456789abcdef"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a api2cart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Api2Cart,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern -complex",+ input: complexPattern,+ want: []string{"1234567890abcdef1234567890abcdef"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("api2cart credentials: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Api2Cart.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Api2Cart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apideck/apideck_integration_test.go b/pkg/detectors/apideck/apideck_integration_test.gonew file mode 100644index 000000000..30eb2fa7c--- /dev/null+++ b/pkg/detectors/apideck/apideck_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package apideck++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApiDeck_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APIDECK_TOKEN")+ inactiveSecret := testSecrets.MustGetField("APIDECK_INACTIVE")+ id := testSecrets.MustGetField("APIDECK_APPID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ApiDeck,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ApiDeck,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ApiDeck.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ApiDeck.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apideck/apideck_test.go b/pkg/detectors/apideck/apideck_test.goindex 30eb2fa7c..268ad1b0b 100644--- a/pkg/detectors/apideck/apideck_test.go+++ b/pkg/detectors/apideck/apideck_test.go@@ -1,121 +1,85 @@-//go:build detectors-// +build detectors-package apideckimport ("context""fmt"+ "strings""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validKeyPattern = "sk_live_abc123def456ghi789jklmno123456pqrstuvwxyz-1234567890123456789012345678901234567890AAui09pU8as"+ validIdPattern = "apideck_id: 1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p7q8r9s0t"+ invalidPattern = strings.ReplaceAll(validKeyPattern, "sk_live", "sk_inactive") + strings.ReplaceAll(validIdPattern, "1", "$"))-func TestApiDeck_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APIDECK_TOKEN")- inactiveSecret := testSecrets.MustGetField("APIDECK_INACTIVE")- id := testSecrets.MustGetField("APIDECK_APPID")+func TestApiDeck_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ApiDeck,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,+ name: "valid pattern",+ input: fmt.Sprintf("credentials: %s \n %s", validKeyPattern, validIdPattern),+ want: []string{+ validKeyPattern + "1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p7q8r9s0t",},- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ApiDeck,- Verified: false,- },- },- wantErr: false,},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("credentials: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ApiDeck.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ApiDeck.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apiflash/apiflash_integration_test.go b/pkg/detectors/apiflash/apiflash_integration_test.gonew file mode 100644index 000000000..c457a4722--- /dev/null+++ b/pkg/detectors/apiflash/apiflash_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package apiflash++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApiflash_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APIFLASH")+ url := testSecrets.MustGetField("API_URL")+ inactiveSecret := testSecrets.MustGetField("APIFLASH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apiflash,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apiflash,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apiflash.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apiflash.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apiflash/apiflash_test.go b/pkg/detectors/apiflash/apiflash_test.goindex c457a4722..cae1a3b72 100644--- a/pkg/detectors/apiflash/apiflash_test.go+++ b/pkg/detectors/apiflash/apiflash_test.go@@ -1,121 +1,84 @@-//go:build detectors-// +build detectors-package apiflashimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ apiflash_key: a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6+ apiflash_url: abc123XYZ456def789ghijklm+ `+ invalidPattern = "0123456789Gbcde^0123456789abcdef")-func TestApiflash_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APIFLASH")- url := testSecrets.MustGetField("API_URL")- inactiveSecret := testSecrets.MustGetField("APIFLASH_INACTIVE")+func TestApiFlash_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apiflash,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("apiflash credentials: %s", validPattern),+ want: []string{"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6abc123XYZ456def789ghijklm"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apiflash,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("apiflash credentials: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apiflash.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apiflash.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apifonica/apifonica_integration_test.go b/pkg/detectors/apifonica/apifonica_integration_test.gonew file mode 100644index 000000000..8df2c4cb5--- /dev/null+++ b/pkg/detectors/apifonica/apifonica_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package apifonica++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApifonica_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APIFONICA_SECRET")+ token := testSecrets.MustGetField("APIFONICA_ID")+ inactiveSecret := testSecrets.MustGetField("APIFONICA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s", secret, token)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ApiFonica,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s but not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ApiFonica,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apifonica.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apifonica.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apifonica/apifonica_test.go b/pkg/detectors/apifonica/apifonica_test.goindex 8df2c4cb5..684ce6556 100644--- a/pkg/detectors/apifonica/apifonica_test.go+++ b/pkg/detectors/apifonica/apifonica_test.go@@ -1,121 +1,81 @@-//go:build detectors-// +build detectors-package apifonicaimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "abc123def45-gh67-ij89-kl01-mn234op56789"+ invalidPattern = "0123456789Gbcde^0123456789abcdef")-func TestApifonica_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APIFONICA_SECRET")- token := testSecrets.MustGetField("APIFONICA_ID")- inactiveSecret := testSecrets.MustGetField("APIFONICA_INACTIVE")+func TestApiFonica_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s", secret, token)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ApiFonica,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("apifonica credentials: %s", validPattern),+ want: []string{"abc123def45-gh67-ij89-kl01-mn234op56789"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s but not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ApiFonica,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("apifonica credentials: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apifonica.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apifonica.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apify/apify_integration_test.go b/pkg/detectors/apify/apify_integration_test.gonew file mode 100644index 000000000..79e866953--- /dev/null+++ b/pkg/detectors/apify/apify_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package apify++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApify_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APIFY")+ inactiveSecret := testSecrets.MustGetField("APIFY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apify secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apify,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apify,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apify.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apify/apify_test.go b/pkg/detectors/apify/apify_test.goindex 79e866953..15798000a 100644--- a/pkg/detectors/apify/apify_test.go+++ b/pkg/detectors/apify/apify_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package apifyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "apify_api_ab12CD34ef56GH78ij90KL12mn34OP56qr78"+ invalidPattern = "apify_api_ab12C#34ef56GH7$ij90KL12mn34OP56qr7)")-func TestApify_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APIFY")- inactiveSecret := testSecrets.MustGetField("APIFY_INACTIVE")+func TestApiFy_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apify secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apify,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apify,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apify.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apilayer/apilayer_integration_test.go b/pkg/detectors/apilayer/apilayer_integration_test.gonew file mode 100644index 000000000..ef4b0fe2f--- /dev/null+++ b/pkg/detectors/apilayer/apilayer_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package apilayer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApilayer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APILAYER")+ inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apilayer,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apilayer,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apilayer/apilayer_test.go b/pkg/detectors/apilayer/apilayer_test.goindex e764f3db1..7c369b411 100644--- a/pkg/detectors/apilayer/apilayer_test.go+++ b/pkg/detectors/apilayer/apilayer_test.go@@ -2,116 +2,79 @@ package apilayerimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "apilayerkey: abcdef1234567890abcdef1234567890"+ invalidPattern = "apilayerkey: abcdef123$5678*0abcde()1234567890)")-func TestApilayer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APILAYER")- inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE")+func TestApiLayer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apilayer,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apilayer,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"abcdef1234567890abcdef1234567890"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apimatic/apimatic.go b/pkg/detectors/apimatic/apimatic.goindex 2d9eb5a07..1198e6707 100644--- a/pkg/detectors/apimatic/apimatic.go+++ b/pkg/detectors/apimatic/apimatic.go@@ -2,17 +2,17 @@ package apimaticimport ("context"- regexp "github.com/wasilibs/go-re2""net/http"- "strings""time"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -23,8 +23,7 @@ var (client = common.SaneHttpClient()// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic"}) + `\b([a-zA-Z0-9]{3,20}@[a-zA-Z0-9]{2,12}.[a-zA-Z0-9]{2,5})\b`)- passPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic"}) + `\b([a-z0-9-\S]{8,32})\b`)+ apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic", "apikey"}) + `\b([a-zA-Z0-9_-]{64})\b`))// Keywords are used for efficiently pre-filtering chunks.@@ -37,47 +36,43 @@ func (s Scanner) Keywords() []string {func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllStringSubmatch(dataStr, -1)- passMatches := passPat.FindAllStringSubmatch(dataStr, -1)+ uniqueApiKeys := make(map[string]struct{})+ for _, matches := range apiKeyPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueApiKeys[matches[1]] = struct{}{}+ }- for _, match := range matches {- if len(match) != 2 {- continue+ for apiKey := range uniqueApiKeys {+ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_APIMatic,+ Raw: []byte(apiKey),}- userPatMatch := strings.TrimSpace(match[1])- for _, idMatch := range passMatches {- if len(idMatch) != 2 {+ if verify {+ timeout := 10 * time.Second+ client.Timeout = timeout++ // api docs: https://docs.apimatic.io/platform-api/#/http/api-endpoints/code-generation-external-apis/list-all-code-generations+ req, err := http.NewRequestWithContext(ctx, "GET", "https://api.apimatic.io/code-generations", http.NoBody)+ if err != nil {continue}- passPatMatch := strings.TrimSpace(idMatch[1])-- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_APIMatic,- Raw: []byte(userPatMatch),- RawV2: []byte(userPatMatch + passPatMatch),- }- if verify {- timeout := 10 * time.Second- client.Timeout = timeout- req, err := http.NewRequestWithContext(ctx, "GET", "https://www.apimatic.io/api/code-generations", nil)- if err != nil {- continue- }- req.SetBasicAuth(userPatMatch, passPatMatch)- res, err := client.Do(req)-- if err == nil {- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- s1.Verified = true- }+ // authentication documentation: https://docs.apimatic.io/platform-api/#/http/guides/authentication+ req.Header.Set("Authorization", "X-Auth-Key "+apiKey)+ resp, err := client.Do(req)+ if err == nil {+ defer resp.Body.Close()+ if resp.StatusCode == 200 {+ s1.Verified = true}+ } else {+ // if any error happens during api request capture that as verificaiton error+ s1.SetVerificationError(err, apiKey)}-- results = append(results, s1)}++ results = append(results, s1)+}return results, nil}diff --git a/pkg/detectors/apimatic/apimatic_integration_test.go b/pkg/detectors/apimatic/apimatic_integration_test.gonew file mode 100644index 000000000..7995bfc48--- /dev/null+++ b/pkg/detectors/apimatic/apimatic_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package apimatic++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAPIMatic_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APIMATIC")+ pass := testSecrets.MustGetField("APIMATIC_PASS")+ inactiveSecret := testSecrets.MustGetField("APIMATIC_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s", secret, pass)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_APIMatic,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s but not valid", inactiveSecret, pass)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_APIMatic,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("APIMatic.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("APIMatic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apimatic/apimatic_test.go b/pkg/detectors/apimatic/apimatic_test.goindex 7995bfc48..09a35794d 100644--- a/pkg/detectors/apimatic/apimatic_test.go+++ b/pkg/detectors/apimatic/apimatic_test.go@@ -1,121 +1,86 @@-//go:build detectors-// +build detectors-package apimaticimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAPIMatic_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APIMATIC")- pass := testSecrets.MustGetField("APIMATIC_PASS")- inactiveSecret := testSecrets.MustGetField("APIMATIC_INACTIVE")+var (+ validPattern = `+ func validateApiMatic() bool {+ apiMaticKey := "rc6iLoUEFGGAWNLsuBJnmsh4tZB-oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"- type args struct {- ctx context.Context- data []byte- verify bool+ return isActive(apiMaticKey, apiMaticPass)}+ `+ invalidPattern = "apiMaticKey: rc6iL$UEFGGAWNLsuBJnmsh4tZB)oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"+)++func TestApiMatic_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s", secret, pass)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_APIMatic,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"rc6iLoUEFGGAWNLsuBJnmsh4tZB-oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s but not valid", inactiveSecret, pass)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_APIMatic,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("APIMatic.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("APIMatic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apimetrics/apimetrics.go b/pkg/detectors/apimetrics/apimetrics.goindex 7cd84d366..98c9ec37f 100644--- a/pkg/detectors/apimetrics/apimetrics.go+++ b/pkg/detectors/apimetrics/apimetrics.go@@ -22,7 +22,7 @@ var (client = common.SaneHttpClient()// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-bA-Z0-9\S]{32})\b`)+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-zA-Z0-9]{32})\b`))// Keywords are used for efficiently pre-filtering chunks.diff --git a/pkg/detectors/apimetrics/apimetrics_integration_test.go b/pkg/detectors/apimetrics/apimetrics_integration_test.gonew file mode 100644index 000000000..a16dc4093--- /dev/null+++ b/pkg/detectors/apimetrics/apimetrics_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package apimetrics++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApiMetrics_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APIMETRICS")+ inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ApiMetrics,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ApiMetrics,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apimetrics/apimetrics_test.go b/pkg/detectors/apimetrics/apimetrics_test.goindex a16dc4093..93e780ba1 100644--- a/pkg/detectors/apimetrics/apimetrics_test.go+++ b/pkg/detectors/apimetrics/apimetrics_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package apimetricsimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "apimetricskey: 5po8TFGawiYNCc1ct4ofWkBqzIfA6IeO"+ invalidPattern = "apimetricskey: A1b2C3d4 E5f6G7h8I9 j0K1l2M3n4O5p6")-func TestApiMetrics_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APIMETRICS")- inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE")+func TestApiMetrics_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ApiMetrics,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"5po8TFGawiYNCc1ct4ofWkBqzIfA6IeO"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ApiMetrics,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apitemplate/apitemplate_integration_test.go b/pkg/detectors/apitemplate/apitemplate_integration_test.gonew file mode 100644index 000000000..ff20dbdbd--- /dev/null+++ b/pkg/detectors/apitemplate/apitemplate_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package apitemplate++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAPITemplate_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APITEMPLATE")+ inactiveSecret := testSecrets.MustGetField("APITEMPLATE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_APITemplate,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_APITemplate,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("APITemplate.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("APITemplate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apitemplate/apitemplate_test.go b/pkg/detectors/apitemplate/apitemplate_test.goindex ff20dbdbd..e19fce97e 100644--- a/pkg/detectors/apitemplate/apitemplate_test.go+++ b/pkg/detectors/apitemplate/apitemplate_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package apitemplateimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "apitemplatekey: a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T"+ invalidPattern = "apitemplatekey: sk_0C2c3D4e5F6@7H8i9J$k1L2m3N4o5P6q7R8s9T")-func TestAPITemplate_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APITEMPLATE")- inactiveSecret := testSecrets.MustGetField("APITEMPLATE_INACTIVE")+func TestApiTemplate_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_APITemplate,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_APITemplate,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("APITemplate.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("APITemplate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apollo/apollo_integration_test.go b/pkg/detectors/apollo/apollo_integration_test.gonew file mode 100644index 000000000..99ea2c755--- /dev/null+++ b/pkg/detectors/apollo/apollo_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package apollo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApollo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APOLLO")+ inactiveSecret := testSecrets.MustGetField("APOLLO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apollo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apollo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apollo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apollo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apollo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apollo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apollo/apollo_test.go b/pkg/detectors/apollo/apollo_test.goindex 99ea2c755..c66ba5417 100644--- a/pkg/detectors/apollo/apollo_test.go+++ b/pkg/detectors/apollo/apollo_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package apolloimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "apollo: A1b2C3d4E5f6G7h8I9j0K1"+ invalidPattern = "apollo: A1b2C3d4-5f6G7h8I9j0K1@$")-func TestApollo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APOLLO")- inactiveSecret := testSecrets.MustGetField("APOLLO_INACTIVE")+func TestApollo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apollo secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apollo,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"A1b2C3d4E5f6G7h8I9j0K1"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apollo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apollo,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apollo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apollo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/appcues/appcues_integration_test.go b/pkg/detectors/appcues/appcues_integration_test.gonew file mode 100644index 000000000..c46205d00--- /dev/null+++ b/pkg/detectors/appcues/appcues_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package appcues++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAppcues_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPCUES")+ user := testSecrets.MustGetField("APPCUES_USER")+ id := testSecrets.MustGetField("APPCUES_ID")+ inactiveSecret := testSecrets.MustGetField("APPCUES_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s", secret, user, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Appcues,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s but not valid", inactiveSecret, user, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Appcues,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Appcues.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Appcues.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/appcues/appcues_test.go b/pkg/detectors/appcues/appcues_test.goindex c46205d00..a239edfac 100644--- a/pkg/detectors/appcues/appcues_test.go+++ b/pkg/detectors/appcues/appcues_test.go@@ -1,122 +1,88 @@-//go:build detectors-// +build detectors-package appcuesimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ appcues abcdef12-3456-7890-1234-56789abcdef0+ appcues abcdef123456789012345678901234567890123+ appcues 12345+ `+ invalidPattern = `+ appcues abcdef12#3456-7890-1234-56789abcdef0+ appcues abcdef12345678901&345678901234567890123+ appcues 123456+ `)-func TestAppcues_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPCUES")- user := testSecrets.MustGetField("APPCUES_USER")- id := testSecrets.MustGetField("APPCUES_ID")- inactiveSecret := testSecrets.MustGetField("APPCUES_INACTIVE")+func TestAppCues_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s", secret, user, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Appcues,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"abcdef12-3456-7890-1234-56789abcdef0abcdef123456789012345678901234567890123"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s but not valid", inactiveSecret, user, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Appcues,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Appcues.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Appcues.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/appfollow/appfollow_integration_test.go b/pkg/detectors/appfollow/appfollow_integration_test.gonew file mode 100644index 000000000..fd6e46745--- /dev/null+++ b/pkg/detectors/appfollow/appfollow_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package appfollow++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAppfollow_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPFOLLOW")+ inactiveSecret := testSecrets.MustGetField("APPFOLLOW_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appfollow secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Appfollow,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appfollow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Appfollow,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Appfollow.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Appfollow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/appfollow/appfollow_test.go b/pkg/detectors/appfollow/appfollow_test.goindex fd6e46745..dbae86cdf 100644--- a/pkg/detectors/appfollow/appfollow_test.go+++ b/pkg/detectors/appfollow/appfollow_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package appfollowimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.333HbjEo1oxVUFcASR0sQ8cMuIJRLcMd5H9iJWDbovCw6ESjNtuoEMQQGPN9aSoxponxrTPvn1.btADhgNetsaUBuwoyHo5ip0Jab6N6MEBnSaT6CHiO6z"+ invalidPattern = "eyQ0eXAiOiJMM1QiLDJhbGciOiJIUXI1NiK7.abcdefgh1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcd1234efgh5678ijkl9012.qwerty12345-ASDFG67890_zxcvb_ABCDE")-func TestAppfollow_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPFOLLOW")- inactiveSecret := testSecrets.MustGetField("APPFOLLOW_INACTIVE")+func TestAppFollow_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appfollow secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Appfollow,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("appfollow credentail: %s", validPattern),+ want: []string{"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.333HbjEo1oxVUFcASR0sQ8cMuIJRLcMd5H9iJWDbovCw6ESjNtuoEMQQGPN9aSoxponxrTPvn1.btADhgNetsaUBuwoyHo5ip0Jab6N6MEBnSaT6CHiO6z"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appfollow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Appfollow,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("appfollow credentail: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Appfollow.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Appfollow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/appointedd/appointedd_integration_test.go b/pkg/detectors/appointedd/appointedd_integration_test.gonew file mode 100644index 000000000..1f2a8dfe6--- /dev/null+++ b/pkg/detectors/appointedd/appointedd_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package appointedd++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAppointedd_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPOINTEDD")+ inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Appointedd,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Appointedd,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/appointedd/appointedd_test.go b/pkg/detectors/appointedd/appointedd_test.goindex 3323032ef..08c498c3c 100644--- a/pkg/detectors/appointedd/appointedd_test.go+++ b/pkg/detectors/appointedd/appointedd_test.go@@ -4,114 +4,78 @@ import ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO=XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9=Jy6yCOXmhbpEZPfY7Y"+ invalidPattern = "Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO^XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9$Jy6yCOXmhbpEZPfY7Y")-func TestAppointedd_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPOINTEDD")- inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE")+func TestAppFollow_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Appointedd,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Appointedd,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("appointedd credentail: %s", validPattern),+ want: []string{"Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO=XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9=Jy6yCOXmhbpEZPfY7Y"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("appointedd credentail: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/appoptics/appoptics_integration_test.go b/pkg/detectors/appoptics/appoptics_integration_test.gonew file mode 100644index 000000000..a50a583b0--- /dev/null+++ b/pkg/detectors/appoptics/appoptics_integration_test.go@@ -0,0 +1,128 @@+//go:build detectors+// +build detectors++package appoptics++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAppoptics_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPOPTICS")+ inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AppOptics,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AppOptics,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/appoptics/appoptics_test.go b/pkg/detectors/appoptics/appoptics_test.goindex a50a583b0..f4c9bd448 100644--- a/pkg/detectors/appoptics/appoptics_test.go+++ b/pkg/detectors/appoptics/appoptics_test.go@@ -1,127 +1,80 @@-//go:build detectors-// +build detectors-package appopticsimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "IABJPR08RmvsGrebJhr1TUdo27-KtTn0mLCKkJJqj5lyba-otXPklygO9DK62o3QSPoIJ4E"+ invalidPattern = "IABJPR08Rmvs.GrebJhr1TUdo27_KtTn0mLCKkJJqj5lyba&otXPklygO9DK62o3QSPoIJ4E")-func TestAppoptics_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPOPTICS")- inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE")+func TestAppOptics_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AppOptics,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("appoptics credentail: %s", validPattern),+ want: []string{"IABJPR08RmvsGrebJhr1TUdo27-KtTn0mLCKkJJqj5lyba-otXPklygO9DK62o3QSPoIJ4E"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AppOptics,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("appoptics credentail: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/appsynergy/appsynergy_integration_test.go b/pkg/detectors/appsynergy/appsynergy_integration_test.gonew file mode 100644index 000000000..363af075e--- /dev/null+++ b/pkg/detectors/appsynergy/appsynergy_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package appsynergy++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAppSynergy_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPSYNERGY")+ inactiveSecret := testSecrets.MustGetField("APPSYNERGY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AppSynergy,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AppSynergy,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AppSynergy.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AppSynergy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/appsynergy/appsynergy_test.go b/pkg/detectors/appsynergy/appsynergy_test.goindex 363af075e..5ca3017f1 100644--- a/pkg/detectors/appsynergy/appsynergy_test.go+++ b/pkg/detectors/appsynergy/appsynergy_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package appsynergyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "mg1pgwlndtq7rbk8i3kum344aso8ggp02ximdhsp8nsqasd3btxf84lz9ekfdpwo"+ invalidPattern = "mG1pgwlndtq7rbk8i3kum344a$o8gGp02ximdhsp8nsqasd3btxf84lz9ekfdpwo")-func TestAppSynergy_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPSYNERGY")- inactiveSecret := testSecrets.MustGetField("APPSYNERGY_INACTIVE")+func TestAppSynergy_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AppSynergy,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("appsynergy credentail: %s", validPattern),+ want: []string{"mg1pgwlndtq7rbk8i3kum344aso8ggp02ximdhsp8nsqasd3btxf84lz9ekfdpwo"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AppSynergy,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("appsynergy credentail: %s", invalidPattern),+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AppSynergy.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AppSynergy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/apptivo/apptivo_integration_test.go b/pkg/detectors/apptivo/apptivo_integration_test.gonew file mode 100644index 000000000..d86c98727--- /dev/null+++ b/pkg/detectors/apptivo/apptivo_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package apptivo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestApptivo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPTIVO")+ id := testSecrets.MustGetField("APPTIVO_KEY")+ inactiveSecret := testSecrets.MustGetField("APPTIVO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apptivo secret %s within apptivo %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apptivo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a apptivo secret %s within but not valid apptivo %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Apptivo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Apptivo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Apptivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/apptivo/apptivo_test.go b/pkg/detectors/apptivo/apptivo_test.goindex d86c98727..b2c74f4c4 100644--- a/pkg/detectors/apptivo/apptivo_test.go+++ b/pkg/detectors/apptivo/apptivo_test.go@@ -1,121 +1,80 @@-//go:build detectors-// +build detectors-package apptivoimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "apptivokey: 1h3l8yjo3pms2h738k8f9094rkltmgy8md-4 - apptivoID: wV2WGN-KUxROVW0mlq8kur7TbKvCTdNc"+ invalidPattern = "apptivokey: 1h3l8yjo3pms2h738k8f9094rkltmgy8md-4")-func TestApptivo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPTIVO")- id := testSecrets.MustGetField("APPTIVO_KEY")- inactiveSecret := testSecrets.MustGetField("APPTIVO_INACTIVE")+func TestApptivo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apptivo secret %s within apptivo %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apptivo,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"1h3l8yjo3pms2h738k8f9094rkltmgy8md-4wV2WGN-KUxROVW0mlq8kur7TbKvCTdNc"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a apptivo secret %s within but not valid apptivo %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Apptivo,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Apptivo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Apptivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/artifactory/artifactory_integration_test.go b/pkg/detectors/artifactory/artifactory_integration_test.gonew file mode 100644index 000000000..deab1d778--- /dev/null+++ b/pkg/detectors/artifactory/artifactory_integration_test.go@@ -0,0 +1,168 @@+//go:build detectors+// +build detectors++package artifactory++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestArtifactory_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ARTIFACTORY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("ARTIFACTORY_INACTIVE")+ appURL := testSecrets.MustGetField("ARTIFACTORY_URL")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s", secret, appURL)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,+ Verified: false,+ }+ r.SetVerificationError(context.DeadlineExceeded)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, real secrets, verification error due to unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,+ Verified: false,+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s but not valid on endpoint %s", inactiveSecret, appURL)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/artifactory/artifactory_test.go b/pkg/detectors/artifactory/artifactory_test.goindex deab1d778..b74c91923 100644--- a/pkg/detectors/artifactory/artifactory_test.go+++ b/pkg/detectors/artifactory/artifactory_test.go@@ -1,167 +1,104 @@-//go:build detectors-// +build detectors-package artifactoryimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl / RVo8ytzB65L.jfrog.io"+ // validPattern2 is for cloud endpoints so it does not have any JFrog endpoint+ validPattern2 = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl"+ invalidPattern = "W0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl^&5YcZhIKwxTdxwpZHf9c1Usu8xNtA / rtest#y$zB65L%.jfrog.io")-func TestArtifactory_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ARTIFACTORY_TOKEN")- inactiveSecret := testSecrets.MustGetField("ARTIFACTORY_INACTIVE")- appURL := testSecrets.MustGetField("ARTIFACTORY_URL")+func TestArtifactory_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ cloudEndpoint string+ useCloudEndpoint bool+ useFoundEndpoint bool+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s", secret, appURL)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, real secrets, verification error due to timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,- Verified: false,- }- r.SetVerificationError(context.DeadlineExceeded)- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, real secrets, verification error due to unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,- Verified: false,- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf("artifactory credentials: %s", validPattern),+ useCloudEndpoint: false,+ useFoundEndpoint: true,+ want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlRVo8ytzB65L.jfrog.io"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a artifactory secret %s but not valid on endpoint %s", inactiveSecret, appURL)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with cloud endpoints",+ input: fmt.Sprintf("artifactory credentials: %s", validPattern2),+ cloudEndpoint: "cloudendpoint.jfrog.io",+ useCloudEndpoint: true,+ useFoundEndpoint: false,+ want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlcloudendpoint.jfrog.io"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("artifactory credentials: %s", invalidPattern),+ useFoundEndpoint: true,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ // this detector use endpoint customizer interface so we need to enable them based on test case+ d.UseFoundEndpoints(test.useFoundEndpoint)+ d.UseCloudEndpoint(test.useCloudEndpoint)+ // if test case provide cloud endpoint use it+ if test.useCloudEndpoint && test.cloudEndpoint != "" {+ d.SetCloudEndpoint(test.cloudEndpoint)+ }++ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/artsy/artsy_integration_test.go b/pkg/detectors/artsy/artsy_integration_test.gonew file mode 100644index 000000000..bf44ba496--- /dev/null+++ b/pkg/detectors/artsy/artsy_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package artsy++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestArtsy_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ARTSY")+ inactiveSecret := testSecrets.MustGetField("ARTSY_INACTIVE")+ id := testSecrets.MustGetField("ARTSY_CLIENTID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Artsy,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Artsy,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Artsy.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Artsy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/artsy/artsy_test.go b/pkg/detectors/artsy/artsy_test.goindex bf44ba496..3ac7c892d 100644--- a/pkg/detectors/artsy/artsy_test.go+++ b/pkg/detectors/artsy/artsy_test.go@@ -1,121 +1,80 @@-//go:build detectors-// +build detectors-package artsyimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "artsy_key: rU0K6hwGw9AeANtXrZ8FQJT9jn4sRdlj - artsy_id: hvQ2fMvUPNczDCdmzi0i"+ invalidPattern = "artsy_key: rU:K6hwGw9AeANtXrZ8F%JT9jn4sR,lj - artsy_id: hvQ2f-vUPNczD+dmzi0i")-func TestArtsy_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ARTSY")- inactiveSecret := testSecrets.MustGetField("ARTSY_INACTIVE")- id := testSecrets.MustGetField("ARTSY_CLIENTID")+func TestArtsy_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Artsy,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"rU0K6hwGw9AeANtXrZ8FQJT9jn4sRdljhvQ2fMvUPNczDCdmzi0i"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Artsy,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Artsy.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Artsy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/asanaoauth/asanaoauth_integration_test.go b/pkg/detectors/asanaoauth/asanaoauth_integration_test.gonew file mode 100644index 000000000..063c55b0c--- /dev/null+++ b/pkg/detectors/asanaoauth/asanaoauth_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package asanaoauth++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAsanaOauth_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ASANAOAUTH_TOKEN")+ inactiveSecret := testSecrets.MustGetField("ASANAOAUTH_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AsanaOauth,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a asana secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AsanaOauth,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AsanaOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ got[i].AnalysisInfo = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AsanaOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/asanaoauth/asanaoauth_test.go b/pkg/detectors/asanaoauth/asanaoauth_test.goindex 063c55b0c..ff9e64ddb 100644--- a/pkg/detectors/asanaoauth/asanaoauth_test.go+++ b/pkg/detectors/asanaoauth/asanaoauth_test.go@@ -1,121 +1,80 @@-//go:build detectors-// +build detectors-package asanaoauthimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "asana_key: v1e0uwkmv2ykvsu998lqe4g7ij2gwtg/xn9vh4jvsokdfaic0bn"+ invalidPattern = "asana_key: v1e0uwkmv2ykvsu998lqe4g-ij2gwtg/xn9vh4jvsokdfaic0bn")-func TestAsanaOauth_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ASANAOAUTH_TOKEN")- inactiveSecret := testSecrets.MustGetField("ASANAOAUTH_INACTIVE")+func TestAsanaOauth_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AsanaOauth,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"v1e0uwkmv2ykvsu998lqe4g7ij2gwtg/xn9vh4jvsokdfaic0bn"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a asana secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AsanaOauth,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AsanaOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil- got[i].AnalysisInfo = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AsanaOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.gonew file mode 100644index 000000000..97a61d918--- /dev/null+++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package asanapersonalaccesstoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ASANA_PAT")+ inactiveSecret := testSecrets.MustGetField("ASANA_PAT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a asana secret %s within but unverified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AsanaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AsanaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.goindex 97a61d918..8cbf64c95 100644--- a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go+++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package asanapersonalaccesstokenimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "asana_token: 594776654034514343561917591881414702593902454625364993/1724908107002616220416212965:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn"+ invalidPattern = "asana_token: 1724908107002616220416212965%594776654034514343561917591881414702593902454625364993:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn-ij2gwtg/xn9vh4jvsokdfaic0bn")-func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ASANA_PAT")- inactiveSecret := testSecrets.MustGetField("ASANA_PAT_INACTIVE")+func TestAsanaPersonalAccessToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"594776654034514343561917591881414702593902454625364993/1724908107002616220416212965:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a asana secret %s within but unverified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AsanaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AsanaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/assemblyai/assemblyai_integration_test.go b/pkg/detectors/assemblyai/assemblyai_integration_test.gonew file mode 100644index 000000000..f042fb413--- /dev/null+++ b/pkg/detectors/assemblyai/assemblyai_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package assemblyai++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAssemblyai_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ASSEMBLYAI")+ inactiveSecret := testSecrets.MustGetField("ASSEMBLYAI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AssemblyAI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AssemblyAI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Assemblyai.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Assemblyai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/assemblyai/assemblyai_test.go b/pkg/detectors/assemblyai/assemblyai_test.goindex f042fb413..5d9dac960 100644--- a/pkg/detectors/assemblyai/assemblyai_test.go+++ b/pkg/detectors/assemblyai/assemblyai_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package assemblyaiimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "assemblyai: mlhekyjhs96mx0r2cxbzky4jzr83fw1q"+ invalidPattern = "assemblyai: Mlhekyjzr83fw1qr2cxbzky4jzr83f1q")-func TestAssemblyai_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ASSEMBLYAI")- inactiveSecret := testSecrets.MustGetField("ASSEMBLYAI_INACTIVE")+func TestAssemblyAI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AssemblyAI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"mlhekyjhs96mx0r2cxbzky4jzr83fw1q"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AssemblyAI,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Assemblyai.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Assemblyai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/atera/atera_integration_test.go b/pkg/detectors/atera/atera_integration_test.gonew file mode 100644index 000000000..ee90eafbc--- /dev/null+++ b/pkg/detectors/atera/atera_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package atera++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAtera_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("ATERA")+ inactiveSecret := testSecrets.MustGetField("ATERA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find an atera secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Atera,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find an atera secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Atera,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Atera.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Atera.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/atera/atera_test.go b/pkg/detectors/atera/atera_test.goindex ee90eafbc..e2000a2d6 100644--- a/pkg/detectors/atera/atera_test.go+++ b/pkg/detectors/atera/atera_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package ateraimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "atera_key: yoo3d5pu3t4zxd6x1vhk7ykmjqarbsv1"+ invalidPattern = "atera_key: yOO3d5pu3t4zxd6x1vhk7ykmjqarbs_1")-func TestAtera_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("ATERA")- inactiveSecret := testSecrets.MustGetField("ATERA_INACTIVE")+func TestAtera_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find an atera secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Atera,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"yoo3d5pu3t4zxd6x1vhk7ykmjqarbsv1"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find an atera secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Atera,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Atera.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Atera.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/audd/audd.go b/pkg/detectors/audd/audd.goindex 7cfde0d94..c55cf2115 100644--- a/pkg/detectors/audd/audd.go+++ b/pkg/detectors/audd/audd.go@@ -3,11 +3,12 @@ package auddimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"diff --git a/pkg/detectors/audd/audd_integration_test.go b/pkg/detectors/audd/audd_integration_test.gonew file mode 100644index 000000000..4f1424150--- /dev/null+++ b/pkg/detectors/audd/audd_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package audd++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAudd_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AUDD")+ inactiveSecret := testSecrets.MustGetField("AUDD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a audd secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Audd,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a audd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Audd,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Audd.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Audd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/audd/audd_test.go b/pkg/detectors/audd/audd_test.goindex 4f1424150..ff38207c6 100644--- a/pkg/detectors/audd/audd_test.go+++ b/pkg/detectors/audd/audd_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package auddimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAudd_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AUDD")- inactiveSecret := testSecrets.MustGetField("AUDD_INACTIVE")+var (+ validPattern = "audd_key: 60fzzcspq2balbxn7f3hi2nvg3h07h4z"+ invalidPattern = "audd_key: 60fuzcsoq2balxbN7f3hi2nvg3h07h4z"+)++func TestAudd_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a audd secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Audd,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"60fzzcspq2balbxn7f3hi2nvg3h07h4z"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a audd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Audd,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Audd.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Audd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.gonew file mode 100644index 000000000..32a0ac059--- /dev/null+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go@@ -0,0 +1,125 @@+//go:build detectors+// +build detectors++package auth0managementapitoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAuth0ManagementApiToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ // use 2592000 for 30 days, this is the maximum allowed+ managementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN")+ inactiveManagementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN_INACTIVE")+ domain := testSecrets.MustGetField("AUTH0_MANAGEMENT_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain %s", managementApiToken, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,+ Redacted: domain,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain https://%s/oauth/token within but not valid", inactiveManagementApiToken, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,+ Redacted: domain,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Auth0ManagementApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Auth0ManagementApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.goindex 32a0ac059..7e098d664 100644--- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go@@ -1,125 +1,114 @@-//go:build detectors-// +build detectors-package auth0managementapitokenimport ("context""fmt"+ "math/rand"+ "strings""testing""time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAuth0ManagementApiToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }+var (+ // TODO(kashif): Refactor the fake token generation if possible+ validPattern = generateRandomString() // this has the exact token string only which can be used in want too+ validDomain = "QHHPu7VPj.sI.auth0.com"+ invalidPattern = `+ auth0_credentails:+ apiToken: eywT2nGMZwOcbsUVBwfiRPEl8P_wnmo6XfdUoGVwxDfOSjNyqhYqFdi.KojZZOM8Ox+ domain: QHHPu7VPj.sI.auth0.com+ `+)- // use 2592000 for 30 days, this is the maximum allowed- managementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN")- inactiveManagementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN_INACTIVE")- domain := testSecrets.MustGetField("AUTH0_MANAGEMENT_DOMAIN")+func TestAuth0ManagementApitToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain %s", managementApiToken, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,- Redacted: domain,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain https://%s/oauth/token within but not valid", inactiveManagementApiToken, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,- Redacted: domain,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern",+ input: makeFakeTokenString(validPattern, validDomain),+ want: []string{validPattern + validDomain},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Auth0ManagementApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}- got[i].Raw = nil}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Auth0ManagementApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }- }- })+// makeFakeTokenString take a string token as parameter and make a string that looks like a token for testing+func makeFakeTokenString(token, domain string) string {+ return fmt.Sprintf("auth0:\n apiToken: %s \n domain: %s", token, domain)+}++// generateRandomString generates exactly 2001 char string for a fake token to by pass the check in detector for testing+func generateRandomString() string {+ const length = 2001+ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-"++ random := rand.New(rand.NewSource(time.Now().UnixNano()))++ var builder strings.Builder+ builder.Grow(length)++ for i := 0; i < length; i++ {+ randomChar := charset[random.Intn(len(charset))]+ builder.WriteByte(randomChar)}++ // append ey in start as the token must start with 'ey'+ return fmt.Sprintf("ey%s", builder.String())}diff --git a/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go b/pkg/detectors/auth0oauth/auth0oauth_integeration_test.gonew file mode 100644index 000000000..5a343487d--- /dev/null+++ b/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go@@ -0,0 +1,125 @@+//go:build detectors+// +build detectors++package auth0oauth++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAuth0oauth_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ domain := testSecrets.MustGetField("AUTH0_DOMAIN")+ clientId := testSecrets.MustGetField("AUTH0_CLIENT_ID") // there is AUTH0_CLIENT_ID2 and AUTH0_CLIENT_SECRET2 pair as well+ clientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET")+ inactiveClientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain %s", clientId, clientSecret, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Auth0oauth,+ Redacted: clientId,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain https://%s/oauth/token within but not valid", clientId, inactiveClientSecret, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Auth0oauth,+ Redacted: clientId,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Auth0oauth.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Auth0oauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/auth0oauth/auth0oauth_test.go b/pkg/detectors/auth0oauth/auth0oauth_test.goindex 5a343487d..e573e5c03 100644--- a/pkg/detectors/auth0oauth/auth0oauth_test.go+++ b/pkg/detectors/auth0oauth/auth0oauth_test.go@@ -1,125 +1,90 @@-//go:build detectors-// +build detectors-package auth0oauthimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAuth0oauth_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }+var (+ validPattern = `+ auth0_credentials file:+ auth0_clientID: kYWr_tL4eYBtqIIvKfSf2-e4T9Cw1CtwE8ufoESVBB7Hi1U+ secret: rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS-MBqIYgxH0vZaL1s5314lgPDLqHXjZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_tTx1kCXOZY_JUy+ domain: 9-KhTIdSopSaMQ2v1YxdFEJN-HNgt7Mn7E8xkfQNqd51AzSGQu2yRaFauth0.com+ `+ invalidPattern = `+ auth0_credentials file:+ auth0_clientID: e4T9Cw1CtwE8ufoESVBB7Hi1U-e4T9Cw1CtwE8ufoESVBB7Hi1U+ secret: MBqIYgxH0vZaL1s5314lgPDLqHX^ZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_+tTx1kCXOZY_JUy-rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS+ domain: 9-KhTIdSopSaMQ2v1YxdFEJN#qd51AzSGQu2yRaFauth1.com+ `+)- domain := testSecrets.MustGetField("AUTH0_DOMAIN")- clientId := testSecrets.MustGetField("AUTH0_CLIENT_ID") // there is AUTH0_CLIENT_ID2 and AUTH0_CLIENT_SECRET2 pair as well- clientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET")- inactiveClientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET_INACTIVE")+func TestAuth0oAuth_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain %s", clientId, clientSecret, domain)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Auth0oauth,- Redacted: clientId,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"kYWr_tL4eYBtqIIvKfSf2-e4T9Cw1CtwE8ufoESVBB7Hi1UrXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS-MBqIYgxH0vZaL1s5314lgPDLqHXjZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_tTx1kCXOZY_JUy"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain https://%s/oauth/token within but not valid", clientId, inactiveClientSecret, domain)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Auth0oauth,- Redacted: clientId,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Auth0oauth.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Auth0oauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/autodesk/autodesk_integration_test.go b/pkg/detectors/autodesk/autodesk_integration_test.gonew file mode 100644index 000000000..d3d90c83c--- /dev/null+++ b/pkg/detectors/autodesk/autodesk_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package autodesk++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAutodesk_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ id := testSecrets.MustGetField("AUTODESK_ID")+ secret := testSecrets.MustGetField("AUTODESK_SECRET")+ inactiveID := testSecrets.MustGetField("AUTODESK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Autodesk,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Autodesk,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Autodesk.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Autodesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/autodesk/autodesk_test.go b/pkg/detectors/autodesk/autodesk_test.goindex d3d90c83c..cf469f817 100644--- a/pkg/detectors/autodesk/autodesk_test.go+++ b/pkg/detectors/autodesk/autodesk_test.go@@ -1,121 +1,80 @@-//go:build detectors-// +build detectors-package autodeskimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "autodesk_key: 2j8Rl67MjoMruYfyIBgGzy2pxcxIQfet / autodesk_secret rHfzZhsSRruLM3Fn"+ invalidPattern = "autodesk: 2mm8Rl67MjoMruYfyIBg5#zy2pxcxIQfet / autodesk_secret RHGklpa")-func TestAutodesk_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- id := testSecrets.MustGetField("AUTODESK_ID")- secret := testSecrets.MustGetField("AUTODESK_SECRET")- inactiveID := testSecrets.MustGetField("AUTODESK_INACTIVE")+func TestAutoDesk_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Autodesk,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"2j8Rl67MjoMruYfyIBgGzy2pxcxIQfetrHfzZhsSRruLM3Fn"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Autodesk,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Autodesk.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Autodesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/autoklose/autoklose_integration_test.go b/pkg/detectors/autoklose/autoklose_integration_test.gonew file mode 100644index 000000000..9675d22cb--- /dev/null+++ b/pkg/detectors/autoklose/autoklose_integration_test.go@@ -0,0 +1,123 @@+//go:build detectors+// +build detectors++package autoklose++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAutoklose_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AUTOKLOSE")+ inactiveSecret := testSecrets.MustGetField("AUTOKLOSE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a autoklose secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Autoklose,+ Verified: true,+ ExtraData: map[string]string{+ "email": "[email protected]",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a autoklose secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Autoklose,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Autoklose.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Autoklose.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/autoklose/autoklose_test.go b/pkg/detectors/autoklose/autoklose_test.goindex 9675d22cb..4ed460f78 100644--- a/pkg/detectors/autoklose/autoklose_test.go+++ b/pkg/detectors/autoklose/autoklose_test.go@@ -1,123 +1,80 @@-//go:build detectors-// +build detectors-package autokloseimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "autoklose credentials: KRXaU9GK3f9yHG1FS-mbwhsIXdW22epH"+ invalidPattern = "autoklose credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH")-func TestAutoklose_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AUTOKLOSE")- inactiveSecret := testSecrets.MustGetField("AUTOKLOSE_INACTIVE")+func TestAutoKlose_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a autoklose secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Autoklose,- Verified: true,- ExtraData: map[string]string{- "email": "[email protected]",- },- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"KRXaU9GK3f9yHG1FS-mbwhsIXdW22epH"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a autoklose secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Autoklose,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Autoklose.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Autoklose.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/autopilot/autopilot_integration_test.go b/pkg/detectors/autopilot/autopilot_integration_test.gonew file mode 100644index 000000000..b15e26cd8--- /dev/null+++ b/pkg/detectors/autopilot/autopilot_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package autopilot++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAutoPilot_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AUTOPILOT")+ inactiveSecret := testSecrets.MustGetField("AUTOPILOT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a autopilot secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AutoPilot,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a autopilot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AutoPilot,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AutoPilot.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AutoPilot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/autopilot/autopilot_test.go b/pkg/detectors/autopilot/autopilot_test.goindex b15e26cd8..dd5a5f67c 100644--- a/pkg/detectors/autopilot/autopilot_test.go+++ b/pkg/detectors/autopilot/autopilot_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package autopilotimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "autopilot credentials: 0fd87cfb1ca6c38c5f1ae5be7b0e395e"+ invalidPattern = "autopilot credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH")-func TestAutoPilot_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AUTOPILOT")- inactiveSecret := testSecrets.MustGetField("AUTOPILOT_INACTIVE")+func TestAutoPilot_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a autopilot secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AutoPilot,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"0fd87cfb1ca6c38c5f1ae5be7b0e395e"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a autopilot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AutoPilot,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AutoPilot.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AutoPilot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.gonew file mode 100644index 000000000..80664332b--- /dev/null+++ b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package avazapersonalaccesstoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAvazaPersonalAccessToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN")+ inactiveSecret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a avaza secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a avaza secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AvazaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AvazaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.goindex 80664332b..7ef07b611 100644--- a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go+++ b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go@@ -1,116 +1,79 @@-//go:build detectors-// +build detectors-package avazapersonalaccesstokenimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAvazaPersonalAccessToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN")- inactiveSecret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN_INACTIVE")+var (+ validPattern = "avaza credentials: 01818612883613176996369293-f113ceb9cf4fa63dc367ab4815b0e1edf890745f"+ invalidPattern = "avaza credentials: 01818612883613176996369293-fzz3ceb0mf4fp63dh367xb4815b0e1edf890745f"+)++func TestAvazaPersonalAccessToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a avaza secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"01818612883613176996369293-f113ceb9cf4fa63dc367ab4815b0e1edf890745f"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a avaza secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AvazaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AvazaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/aviationstack/aviationstack_integration_test.go b/pkg/detectors/aviationstack/aviationstack_integration_test.gonew file mode 100644index 000000000..b716fcd19--- /dev/null+++ b/pkg/detectors/aviationstack/aviationstack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package aviationstack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAviationStack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AVIATIONSTACK")+ inactiveSecret := testSecrets.MustGetField("AVIATIONSTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AviationStack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AviationStack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AviationStack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AviationStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/aviationstack/aviationstack_test.go b/pkg/detectors/aviationstack/aviationstack_test.goindex b716fcd19..c76924c9a 100644--- a/pkg/detectors/aviationstack/aviationstack_test.go+++ b/pkg/detectors/aviationstack/aviationstack_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package aviationstackimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "aviationstack credentials: osh0kjinsc2atoaqntoy1hdjppg54449"+ invalidPattern = "aviationstack credentials: OSh0lMjinsc2atoaqnto[]1hdjppg5449")-func TestAviationStack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AVIATIONSTACK")- inactiveSecret := testSecrets.MustGetField("AVIATIONSTACK_INACTIVE")+func TestAviationStack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AviationStack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"osh0kjinsc2atoaqntoy1hdjppg54449"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AviationStack,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AviationStack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AviationStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/aws/aws_integration_test.go b/pkg/detectors/aws/aws_integration_test.gonew file mode 100644index 000000000..92ffa5c3c--- /dev/null+++ b/pkg/detectors/aws/aws_integration_test.go@@ -0,0 +1,449 @@+//go:build detectors+// +build detectors++package aws++import (+ "context"+ "crypto/sha256"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/stretchr/testify/assert"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX"++var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)++// Our AWS detector interacts with AWS in an (expectedly) uncommon way that triggers some odd AWS behavior. (This odd+// behavior doesn't affect "normal" AWS use, so it's not really "broken" - it's just something that we have to work+// around.) The AWS detector code has a long comment explaining this in more detail, but the basic issue is that AWS STS+// is stateful, so the behavior of these tests can vary depending on which of them you run, and in which order. This+// particular test (TestAWS_FromChunk_InvalidValidReuseIDSequence) duplicates some logic in the "big" test table in the+// other test in this file, but extracting it in this way as well makes it fail more consistently when it's supposed to+// fail, which is why it's extracted.+func TestAWS_FromChunk_InvalidValidReuseIDSequence(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AWS")+ id := testSecrets.MustGetField("AWS_ID")+ inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")++ d := scanner{}++ ignoreOpts := []cmp.Option{cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")}++ got, err := d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, inactiveSecret)))+ if assert.NoError(t, err) {+ want := []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ },+ },+ }+ if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {+ t.Errorf("AWS.FromData() (valid ID, invalid secret) diff: (-got +want)\n%s", diff)+ }+ }++ got, err = d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, secret)))+ if assert.NoError(t, err) {+ want := []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: true,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",+ },+ },+ }+ if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {+ t.Errorf("AWS.FromData() (valid secret after invalid secret using same ID) diff: (-got +want)\n%s", diff)+ }+ }+}++func TestAWS_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AWS")+ id := testSecrets.MustGetField("AWS_ID")+ inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")+ inactiveID := id[:len(id)-3] + "XYZ"+ hasher := sha256.New()+ hasher.Write([]byte(inactiveSecret))+ hash := string(hasher.Sum(nil))++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationError bool+ }{+ {+ name: "found, verified",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: true,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: scanner{verificationClient: unverifiedSecretClient},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "found two, one included for every ID found",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIAZAVB57H55F3T4XYZ",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ },+ },+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: true,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "not found, because unverified secret was a hash",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: true,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",+ },+ },+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: inactiveID,+ ExtraData: map[string]string{+ "account": "619888638459",+ "resource_type": "Access key",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified, with leading +",+ s: scanner{+ verificationClient: unverifiedSecretClient,+ },+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "skipped",+ s: scanner{+ skipIDs: map[string]struct{}{+ "AKIAZAVB57H55F3T4BKH": {},+ },+ },+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ wantErr: false,+ },+ {+ name: "found, would be verified if not for http timeout",+ s: scanner{+ verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),+ },+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ },+ },+ },+ wantErr: false,+ wantVerificationError: true,+ },+ {+ name: "found, unverified due to unexpected http response status",+ s: scanner{+ verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),+ },+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ },+ },+ },+ wantErr: false,+ wantVerificationError: true,+ },+ {+ name: "found, unverified due to invalid aws_secret with valid canary access_key_id",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: canaryAccessKeyID,+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "171436882533",+ "is_canary": "true",+ "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",+ },+ },+ },+ wantErr: false,+ wantVerificationError: false,+ },+ {+ name: "found, valid canary token with no verification",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)),+ verify: false,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: canaryAccessKeyID,+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "171436882533",+ "is_canary": "true",+ "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",+ },+ },+ },+ wantErr: false,+ wantVerificationError: false,+ },+ {+ name: "verified secret checked directly after unverified secret with same key id",+ s: scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: true,+ Redacted: "AKIAZAVB57H55F3T4BKH",+ ExtraData: map[string]string{+ "resource_type": "Access key",+ "account": "619888638459",+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",+ },+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := tt.s+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationError {+ t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())+ }+ }+ ignoreOpts := []cmp.Option{+ cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"),+ cmpopts.SortSlices(func(x, y detectors.Result) bool {+ return x.Redacted < y.Redacted+ }),+ }+ if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {+ t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.goindex 92ffa5c3c..576451fed 100644--- a/pkg/detectors/aws/aws_test.go+++ b/pkg/detectors/aws/aws_test.go@@ -1,448 +1,89 @@-//go:build detectors-// +build detectors-package awsimport ("context"- "crypto/sha256"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/stretchr/testify/assert"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX"--var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)--// Our AWS detector interacts with AWS in an (expectedly) uncommon way that triggers some odd AWS behavior. (This odd-// behavior doesn't affect "normal" AWS use, so it's not really "broken" - it's just something that we have to work-// around.) The AWS detector code has a long comment explaining this in more detail, but the basic issue is that AWS STS-// is stateful, so the behavior of these tests can vary depending on which of them you run, and in which order. This-// particular test (TestAWS_FromChunk_InvalidValidReuseIDSequence) duplicates some logic in the "big" test table in the-// other test in this file, but extracting it in this way as well makes it fail more consistently when it's supposed to-// fail, which is why it's extracted.-func TestAWS_FromChunk_InvalidValidReuseIDSequence(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AWS")- id := testSecrets.MustGetField("AWS_ID")- inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")-- d := scanner{}-- ignoreOpts := []cmp.Option{cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")}-- got, err := d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, inactiveSecret)))- if assert.NoError(t, err) {- want := []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- },- },+var (+ validPattern = `+ aws credentials{+ id: ABIAS9L8MS5IPHTZPPUQ+ secret: .v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63;}- if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {- t.Errorf("AWS.FromData() (valid ID, invalid secret) diff: (-got +want)\n%s", diff)+ `+ invalidPattern = `+ aws credentials{+ id: AKIAs9L8MS5iPHTZPPUQ+ secret: $YenOG.PKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63;}- }-- got, err = d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, secret)))- if assert.NoError(t, err) {- want := []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: true,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",- "user_id": "AIDAZAVB57H5V3Q4ACRGM",- },- },- }- if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {- t.Errorf("AWS.FromData() (valid secret after invalid secret using same ID) diff: (-got +want)\n%s", diff)- }- }-}+ `+)-func TestAWS_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AWS")- id := testSecrets.MustGetField("AWS_ID")- inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")- inactiveID := id[:len(id)-3] + "XYZ"- hasher := sha256.New()- hasher.Write([]byte(inactiveSecret))- hash := string(hasher.Sum(nil))+func TestAWS_Pattern(t *testing.T) {+ d := scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s scanner- args args- want []detectors.Result- wantErr bool- wantVerificationError bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: true,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",- "user_id": "AIDAZAVB57H5V3Q4ACRGM",- },- },- },- wantErr: false,- },- {- name: "found, unverified",- s: scanner{verificationClient: unverifiedSecretClient},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- },- },- },- wantErr: false,- },- {- name: "not found",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- {- name: "found two, one included for every ID found",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: "AKIAZAVB57H55F3T4XYZ",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- },- },- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: true,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",- "user_id": "AIDAZAVB57H5V3Q4ACRGM",- },- },- },- wantErr: false,- },- {- name: "not found, because unverified secret was a hash",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"ABIAS9L8MS5IPHTZPPUQv2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63"},},{- name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: true,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",- "user_id": "AIDAZAVB57H5V3Q4ACRGM",- },- },- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: inactiveID,- ExtraData: map[string]string{- "account": "619888638459",- "resource_type": "Access key",- },- },- },- wantErr: false,- },- {- name: "found, unverified, with leading +",- s: scanner{- verificationClient: unverifiedSecretClient,- },- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- },- },- },- wantErr: false,- },- {- name: "skipped",- s: scanner{- skipIDs: map[string]struct{}{- "AKIAZAVB57H55F3T4BKH": {},- },- },- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- wantErr: false,- },- {- name: "found, would be verified if not for http timeout",- s: scanner{- verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),- },- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- },- },- },- wantErr: false,- wantVerificationError: true,- },- {- name: "found, unverified due to unexpected http response status",- s: scanner{- verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),- },- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- },- },- },- wantErr: false,- wantVerificationError: true,- },- {- name: "found, unverified due to invalid aws_secret with valid canary access_key_id",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: canaryAccessKeyID,- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "171436882533",- "is_canary": "true",- "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",- },- },- },- wantErr: false,- wantVerificationError: false,- },- {- name: "found, valid canary token with no verification",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)),- verify: false,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: canaryAccessKeyID,- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "171436882533",- "is_canary": "true",- "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",- },- },- },- wantErr: false,- wantVerificationError: false,- },- {- name: "verified secret checked directly after unverified secret with same key id",- s: scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AWS,- Verified: true,- Redacted: "AKIAZAVB57H55F3T4BKH",- ExtraData: map[string]string{- "resource_type": "Access key",- "account": "619888638459",- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",- "user_id": "AIDAZAVB57H5V3Q4ACRGM",- },- },- },- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := tt.s- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if (got[i].VerificationError() != nil) != tt.wantVerificationError {- t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}+ return}- ignoreOpts := []cmp.Option{- cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"),- cmpopts.SortSlices(func(x, y detectors.Result) bool {- return x.Redacted < y.Redacted- }),++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }}- if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {- t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/awssessionkeys/awssessionkeys_test.go b/pkg/detectors/awssessionkeys/awssessionkeys_test.gonew file mode 100644index 000000000..8ed4028e6--- /dev/null+++ b/pkg/detectors/awssessionkeys/awssessionkeys_test.go@@ -0,0 +1,91 @@+package awssessionkey++import (+ "context"+ "testing"++ "github.com/google/go-cmp/cmp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ aws credentials{+ id: ASIABBKK02W42Q3IPSPG+ secret: fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O+ session: >aSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa~+ }+ `+ invalidPattern = `+ aws credentials{+ id: ASIABBKK02W42Q3IPSPG+ secret: $YenOG.PKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63;+ }+ `+)++func TestAWSSessionKey_Pattern(t *testing.T) {+ d := scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})++ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "valid pattern",+ input: validPattern,+ want: []string{"ASIABBKK02W42Q3IPSPGfkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8OaSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa"},+ },+ {+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}diff --git a/pkg/detectors/axonaut/axonaut_integration_test.go b/pkg/detectors/axonaut/axonaut_integration_test.gonew file mode 100644index 000000000..0d64cbc64--- /dev/null+++ b/pkg/detectors/axonaut/axonaut_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package axonaut++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAxonaut_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AXONAUT")+ inactiveSecret := testSecrets.MustGetField("AXONAUT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a axonaut secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Axonaut,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a axonaut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Axonaut,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Axonaut.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Axonaut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/axonaut/axonaut_test.go b/pkg/detectors/axonaut/axonaut_test.goindex 0d64cbc64..fdf924afc 100644--- a/pkg/detectors/axonaut/axonaut_test.go+++ b/pkg/detectors/axonaut/axonaut_test.go@@ -1,120 +1,88 @@-//go:build detectors-// +build detectors-package axonautimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ axonaut credentials{+ key: 4ve4aj6v38uiadaq9hcgpupp2b3lh2k8+ }+ `+ invalidPattern = `+ axonaut credentials{+ key: ASIABBKK02W42Q3IPSPG+ }+ `)-func TestAxonaut_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AXONAUT")- inactiveSecret := testSecrets.MustGetField("AXONAUT_INACTIVE")+func TestAxonaut_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a axonaut secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Axonaut,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"4ve4aj6v38uiadaq9hcgpupp2b3lh2k8"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a axonaut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Axonaut,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Axonaut.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Axonaut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/aylien/aylien_integration_test.go b/pkg/detectors/aylien/aylien_integration_test.gonew file mode 100644index 000000000..e1315f5f9--- /dev/null+++ b/pkg/detectors/aylien/aylien_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package aylien++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAylien_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AYLIEN")+ id := testSecrets.MustGetField("AYLIEN_ID")+ inactiveSecret := testSecrets.MustGetField("AYLIEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aylien,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Aylien,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Aylien.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Aylien.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/aylien/aylien_test.go b/pkg/detectors/aylien/aylien_test.goindex e1315f5f9..5f01ae03b 100644--- a/pkg/detectors/aylien/aylien_test.go+++ b/pkg/detectors/aylien/aylien_test.go@@ -1,121 +1,88 @@-//go:build detectors-// +build detectors-package aylienimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ aylien credentials:+ aylien key: cr479du2l9pkmhar8gw5hufofvwp86q9+ aylien id: y3ejw028+ `+ invalidPattern = `+ aylien credentials:+ aylien key: cr4U9du2l9pkmhar8gw5hufofvWp86q9+ aylien id: y3ejwA8+ `)-func TestAylien_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AYLIEN")- id := testSecrets.MustGetField("AYLIEN_ID")- inactiveSecret := testSecrets.MustGetField("AYLIEN_INACTIVE")+func TestAylien_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aylien,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"cr479du2l9pkmhar8gw5hufofvwp86q9y3ejw028"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Aylien,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Aylien.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Aylien.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/ayrshare/ayrshare_integration_test.go b/pkg/detectors/ayrshare/ayrshare_integration_test.gonew file mode 100644index 000000000..da11fac95--- /dev/null+++ b/pkg/detectors/ayrshare/ayrshare_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package ayrshare++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAyrshare_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AYRSHARE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("AYRSHARE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ayrshare,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Ayrshare,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Ayrshare.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Ayrshare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/ayrshare/ayrshare_test.go b/pkg/detectors/ayrshare/ayrshare_test.goindex da11fac95..9eb55166f 100644--- a/pkg/detectors/ayrshare/ayrshare_test.go+++ b/pkg/detectors/ayrshare/ayrshare_test.go@@ -1,120 +1,80 @@-//go:build detectors-// +build detectors-package ayrshareimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = "ayrshare credentials: 2FTJTA1C-BXO0DV4J-HGTP9E62-QHQSILY1"+ invalidPattern = "ayrshare credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH")-func TestAyrshare_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AYRSHARE_TOKEN")- inactiveSecret := testSecrets.MustGetField("AYRSHARE_INACTIVE")+func TestAyrShare_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ayrshare,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"2FTJTA1C-BXO0DV4J-HGTP9E62-QHQSILY1"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Ayrshare,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Ayrshare.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Ayrshare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/azure/azure_integration_test.go b/pkg/detectors/azure/azure_integration_test.gonew file mode 100644index 000000000..9ecc18ab1--- /dev/null+++ b/pkg/detectors/azure/azure_integration_test.go@@ -0,0 +1,113 @@+//go:build detectors+// +build detectors++package azure++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzure_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_SECRET")+ secretInactive := testSecrets.MustGetField("AZURE_INACTIVE")+ id := testSecrets.MustGetField("AZURE_ID")+ tenantId := testSecrets.MustGetField("AZURE_TENANT_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`+ tenant_id=%s+ client_id=%s+ client_secret=%s+ client_secret=%s+ `, tenantId, id, secretInactive, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Azure,+ Redacted: id,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`+ tenant_id=%s+ client_id=%s+ client_secret=%s+ `, tenantId, id, secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Azure,+ Redacted: id,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}diff --git a/pkg/detectors/azure/azure_test.go b/pkg/detectors/azure/azure_test.goindex 9ecc18ab1..74b2fadb5 100644--- a/pkg/detectors/azure/azure_test.go+++ b/pkg/detectors/azure/azure_test.go@@ -1,112 +1,89 @@-//go:build detectors-// +build detectors-package azureimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAzure_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AZURE_SECRET")- secretInactive := testSecrets.MustGetField("AZURE_INACTIVE")- id := testSecrets.MustGetField("AZURE_ID")- tenantId := testSecrets.MustGetField("AZURE_TENANT_ID")+var (+ validPattern = `+ azure credentials:+ azureClientID: clientid9304d5df4-aac1-6117-552c-7f70c89a40d9+ azureTenant: tenant_idid9304d5df4-aac1-6117-552c-7f70c89a40d9+ azureClientSecret: clientsecretY_0w|[cGpan41k6ng.ol414sp4ccw2v_rkfmbs537i+ `+ invalidPattern = `+ azure credentials:+ azureClientID: 9304d5df4-aac1-6117-552c-7f70c89a+ azureTenant: id9304d5df4-aac1-6117-55-7f70c89a40d9+ azureClientSecret: Y_0w|[cGpan41k6ng.+ `+)++func TestAzure_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`- tenant_id=%s- client_id=%s- client_secret=%s- client_secret=%s- `, tenantId, id, secretInactive, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Azure,- Redacted: id,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`- tenant_id=%s- client_id=%s- client_secret=%s- `, tenantId, id, secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Azure,- Redacted: id,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"304d5df4-aac1-6117-552c-7f70c89a40d9cGpan41k6ng.ol414sp4ccw2v_rkfmbs53304d5df4-aac1-6117-552c-7f70c89a40d9"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/azurebatch/azurebatch_integration_test.go b/pkg/detectors/azurebatch/azurebatch_integration_test.gonew file mode 100644index 000000000..dc92a401b--- /dev/null+++ b/pkg/detectors/azurebatch/azurebatch_integration_test.go@@ -0,0 +1,130 @@+//go:build detectors+// +build detectors++package azurebatch++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzurebatch_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ url := testSecrets.MustGetField("AZUREBATCH_URL")+ secret := testSecrets.MustGetField("AZUREBATCH_KEY")+ inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureBatch,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureBatch,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")++ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azurebatch/azurebatch_test.go b/pkg/detectors/azurebatch/azurebatch_test.goindex dc92a401b..82a5ac19b 100644--- a/pkg/detectors/azurebatch/azurebatch_test.go+++ b/pkg/detectors/azurebatch/azurebatch_test.go@@ -1,130 +1,88 @@-//go:build detectors-// +build detectors-package azurebatchimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ azure batch:+ url: https://JrxlYxT+0hW.YSA.batch.azure.com+ secret: BXIMbhBlC3=5hIbqCEKvq7opaV2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/LB0+ `+ invalidPattern = `+ azure batch:+ url: http://invalid.this.batch.azure.com+ secret: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/+ `)-func TestAzurebatch_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- url := testSecrets.MustGetField("AZUREBATCH_URL")- secret := testSecrets.MustGetField("AZUREBATCH_KEY")- inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE")+func TestAzureBatch_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureBatch,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureBatch,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"https://JrxlYxT+0hW.YSA.batch.azure.comBXIMbhBlC3=5hIbqCEKvq7opaV2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/LB0"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())- }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.gonew file mode 100644index 000000000..8ed032f97--- /dev/null+++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go@@ -0,0 +1,129 @@+//go:build detectors+// +build detectors++package azurecontainerregistry++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzureContainerRegistry_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ azureHost := testSecrets.MustGetField("AZURE_CR_HOST")+ password := testSecrets.MustGetField("AZURE_CR_PASSWORD")+ passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureContainerRegistry,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureContainerRegistry,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.goindex 8ed032f97..b477796ab 100644--- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go+++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go@@ -1,128 +1,87 @@-//go:build detectors-// +build detectors-package azurecontainerregistryimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ azure:+ url: 02r0TG8OBkyxWIhSrcCMUQCazXEpVwdrTSVKCstwytZjAa4b2fKJFCy9n.azurecr.io+ secret: np5kXaH6Qu9zVE5nudGcsm+3/qSOWt/hQ/4bP0kvk6aw98pJEjE3+ `+ invalidPattern = `+ azure:+ url: http://invalid.azurecr.io.azure.com+ secret: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/+ `)-func TestAzureContainerRegistry_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- azureHost := testSecrets.MustGetField("AZURE_CR_HOST")- password := testSecrets.MustGetField("AZURE_CR_PASSWORD")- passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE")+func TestAzureContainerRegistry_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureContainerRegistry,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"02r0TG8OBkyxWIhSrcCMUQCazXEpVwdrTSVKCstwytZjAa4b2fKJFCy9n.azurecr.ionp5kXaH6Qu9zVE5nudGcsm+3/qSOWt/hQ/4bP0kvk6aw98pJEjE3"},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureContainerRegistry,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.gonew file mode 100644index 000000000..2b7c68f3f--- /dev/null+++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go@@ -0,0 +1,170 @@+//go:build detectors+// +build detectors++package azuredevopspersonalaccesstoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzureDevopsPersonalAccessToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_DEVOPS_PAT")+ inactiveSecret := testSecrets.MustGetField("AZURE_DEVOPS_PAT_INACTIVE")+ org := testSecrets.MustGetField("AZURE_DEVOPS_PAT_ORG")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,+ Verified: true,+ RawV2: []byte(secret + org),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,+ Verified: false,+ RawV2: []byte(inactiveSecret + org),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,+ Verified: false,+ RawV2: []byte(secret + org),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,+ Verified: false,+ RawV2: []byte(secret + org),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureDevopsPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if len(got[i].RawV2) == 0 {+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureDevopsPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.goindex 2b7c68f3f..b19ef9de3 100644--- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go+++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go@@ -1,169 +1,87 @@-//go:build detectors-// +build detectors-package azuredevopspersonalaccesstokenimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ azure:+ azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un+ azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG+ `+ invalidPattern = `+ azure:+ azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un+ azure_org_id: LOKi+ `)-func TestAzureDevopsPersonalAccessToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AZURE_DEVOPS_PAT")- inactiveSecret := testSecrets.MustGetField("AZURE_DEVOPS_PAT_INACTIVE")- org := testSecrets.MustGetField("AZURE_DEVOPS_PAT_ORG")+func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,- Verified: true,- RawV2: []byte(secret + org),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,- Verified: false,- RawV2: []byte(inactiveSecret + org),- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,- Verified: false,- RawV2: []byte(secret + org),- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,- Verified: false,- RawV2: []byte(secret + org),- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AzureDevopsPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if len(got[i].RawV2) == 0 {- t.Fatalf("no rawV2 secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AzureDevopsPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go b/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.gonew file mode 100644index 000000000..2d9881ed7--- /dev/null+++ b/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go@@ -0,0 +1,170 @@+//go:build detectors+// +build detectors++package azurefunctionkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzureFunctionKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_FUNCTION_KEY")+ inactiveSecret := testSecrets.MustGetField("AZURE_FUNCTION_KEY_INACTIVE")+ url := testSecrets.MustGetField("AZURE_FUNCTION_URL")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,+ Verified: true,+ RawV2: []byte(secret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,+ Verified: false,+ RawV2: []byte(inactiveSecret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,+ Verified: false,+ RawV2: []byte(secret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,+ Verified: false,+ RawV2: []byte(secret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureFunctionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if len(got[i].RawV2) == 0 {+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureFunctionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go b/pkg/detectors/azurefunctionkey/azurefunctionkey_test.goindex 2d9881ed7..f284a28d7 100644--- a/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go+++ b/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go@@ -1,169 +1,90 @@-//go:build detectors-// +build detectors-package azurefunctionkeyimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ azure:+ azureURL: https://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E+ azureFunctionkey: B8sm0KyfL1y8vPH3IDTdefevHBCGK33-=+ `+ invalidPattern = `+ azure:+ azureURL: http://invalid.azurecr.io.azure.com+ azureFunctionkey: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/+ `)-func TestAzureFunctionKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AZURE_FUNCTION_KEY")- inactiveSecret := testSecrets.MustGetField("AZURE_FUNCTION_KEY_INACTIVE")- url := testSecrets.MustGetField("AZURE_FUNCTION_URL")+func TestAzureFunctionKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureFunctionKey,- Verified: true,- RawV2: []byte(secret + url),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation- verify: true,+ name: "valid pattern",+ input: validPattern,+ want: []string{+ "azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0Ehttps://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E",+ "B8sm0KyfL1y8vPH3IDTdefevHBCGK33https://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E",},- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureFunctionKey,- Verified: false,- RawV2: []byte(inactiveSecret + url),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureFunctionKey,- Verified: false,- RawV2: []byte(secret + url),- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureFunctionKey,- Verified: false,- RawV2: []byte(secret + url),- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AzureFunctionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if len(got[i].RawV2) == 0 {- t.Fatalf("no rawV2 secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AzureFunctionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.gonew file mode 100644index 000000000..c59e16cee--- /dev/null+++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go@@ -0,0 +1,170 @@+//go:build detectors+// +build detectors++package azuresearchadminkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzureSearchAdminKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY")+ inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_INACTIVE")+ service := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_SERVICE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,+ Verified: true,+ RawV2: []byte(secret + service),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within but not valid", inactiveSecret, service)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,+ Verified: false,+ RawV2: []byte(inactiveSecret + service),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,+ Verified: false,+ RawV2: []byte(secret + service),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,+ Verified: false,+ RawV2: []byte(secret + service),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureSearchAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if len(got[i].RawV2) == 0 {+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureSearchAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.goindex c59e16cee..aeb5b2094 100644--- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go+++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go@@ -1,169 +1,87 @@-//go:build detectors-// +build detectors-package azuresearchadminkeyimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ validPattern = `+ azure:+ azureKey: wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJyma+ azureService: TestingService01+ `+ invalidPattern = `+ azure:+ Key: wRRPyhjv8m6JGRujUUr-PK#a8d3rJ0mrGAxhmqf3A68OgZmlWUJyma+ Service: TS01+ `)-func TestAzureSearchAdminKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY")- inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_INACTIVE")- service := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_SERVICE")+func TestAzureSearchAdminKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,- Verified: true,- RawV2: []byte(secret + service),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within but not valid", inactiveSecret, service)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,- Verified: false,- RawV2: []byte(inactiveSecret + service),- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaTestingService01", "wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaazureKey"},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,- Verified: false,- RawV2: []byte(secret + service),- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,- Verified: false,- RawV2: []byte(secret + service),- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AzureSearchAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if len(got[i].RawV2) == 0 {- t.Fatalf("no rawV2 secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AzureSearchAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.gonew file mode 100644index 000000000..1967a9b97--- /dev/null+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go@@ -0,0 +1,170 @@+//go:build detectors+// +build detectors++package azuresearchquerykey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzureSearchQueryKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY")+ inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_INACTIVE")+ url := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_URL")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,+ Verified: true,+ RawV2: []byte(secret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,+ Verified: false,+ RawV2: []byte(inactiveSecret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,+ Verified: false,+ RawV2: []byte(secret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,+ Verified: false,+ RawV2: []byte(secret + url),+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureSearchQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if len(got[i].RawV2) == 0 {+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureSearchQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.goindex 1967a9b97..3af2254ec 100644--- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go@@ -1,169 +1,88 @@-//go:build detectors-// +build detectors-package azuresearchquerykeyimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++var (+ validPattern = `+ azure:+ azure_url: https://tzyexx2ktdfhha8w1cktqzbrgv37ywtu.search.windows.net/indexes/n81wg81jogjfq93cyxfi67vy2g7vwlcqfgi+ azure_key: OKalbM5EBt5hloqU46phTUCZqvNAlZ4S2Jd2gFUCOQ3HG0vQ2uEp- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ `+ invalidPattern = `+ azure:+ url: http://invalid.azurecr.io.azure.com+ azure_key: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/+ `)-func TestAzureSearchQueryKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY")- inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_INACTIVE")- url := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_URL")+func TestAzureSearchQueryKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,- Verified: true,- RawV2: []byte(secret + url),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,- Verified: false,- RawV2: []byte(inactiveSecret + url),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"OKalbM5EBt5hloqU46phTUCZqvNAlZ4S2Jd2gFUCOQ3HG0vQ2uEphttps://tzyexx2ktdfhha8w1cktqzbrgv37ywtu.search.windows.net/indexes/n81wg81jogjfq93cyxfi67vy2g7vwlcqfgi"},},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,- Verified: false,- RawV2: []byte(secret + url),- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,- Verified: false,- RawV2: []byte(secret + url),- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AzureSearchQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if len(got[i].RawV2) == 0 {- t.Fatalf("no rawV2 secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("AzureSearchQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/azurestorage/azurestorage_integration_test.go b/pkg/detectors/azurestorage/azurestorage_integration_test.gonew file mode 100644index 000000000..1e45f122e--- /dev/null+++ b/pkg/detectors/azurestorage/azurestorage_integration_test.go@@ -0,0 +1,205 @@+//go:build detectors+// +build detectors++package azurestorage++import (+ "context"+ "fmt"+ "strings"+ "testing"+ "time"++ regexp "github.com/wasilibs/go-re2"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzurestorage_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("AZURE_STORAGE")+ inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE")++ accountNamePat := regexp.MustCompile(`AccountName=(?P<account_name>[^;]+);AccountKey`)+ accountName := accountNamePat.FindStringSubmatch(secret)[1]+ validKeyInvalidAccountName := strings.Replace(secret, accountName, "invalid", 1)++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureStorage,+ Verified: true,+ ExtraData: map[string]string{+ "account_name": "teststoragebytruffle",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureStorage,+ Verified: false,+ ExtraData: map[string]string{+ "account_name": "teststoragebytruffle",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_AzureStorage,+ Verified: false,+ ExtraData: map[string]string{+ "account_name": "teststoragebytruffle",+ },+ }+ r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),+ verify: true,+ },+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_AzureStorage,+ Verified: false,+ ExtraData: map[string]string{+ "account_name": "teststoragebytruffle",+ },+ }+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found secret with invalid account name",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", validKeyInvalidAccountName)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureStorage,+ Verified: false,+ ExtraData: map[string]string{+ "account_name": "invalid",+ },+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azurestorage/azurestorage_test.go b/pkg/detectors/azurestorage/azurestorage_test.goindex 343f65390..676900289 100644--- a/pkg/detectors/azurestorage/azurestorage_test.go+++ b/pkg/detectors/azurestorage/azurestorage_test.go@@ -1,203 +1,85 @@-//go:build detectors-// +build detectors-package azurestorageimport ("context"- "fmt"- regexp "github.com/wasilibs/go-re2"- "strings""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAzurestorage_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("AZURE_STORAGE")- inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE")+var (+ validPattern = `+ azure storage:+ DefaultEndpointsProtocol=https;AccountName=myStorageAccount;AccountKey=myAccountKey12345==;EndpointSuffix=core.windows.net+ `+ invalidPattern = `+ azure storage:+ DefaultEndpointsProtocol=https;AccountName=;AccountKey=myAccountKey12345==;EndpointSuffix=core.ubuntu.net+ `+)- accountNamePat := regexp.MustCompile(`AccountName=(?P<account_name>[^;]+);AccountKey`)- accountName := accountNamePat.FindStringSubmatch(secret)[1]- validKeyInvalidAccountName := strings.Replace(secret, accountName, "invalid", 1)+func TestAzureStorage_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureStorage,- Verified: true,- ExtraData: map[string]string{- "account_name": "teststoragebytruffle",- },- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureStorage,- Verified: false,- ExtraData: map[string]string{- "account_name": "teststoragebytruffle",- },- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_AzureStorage,- Verified: false,- ExtraData: map[string]string{- "account_name": "teststoragebytruffle",- },- }- r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret)- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),- verify: true,- },- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_AzureStorage,- Verified: false,- ExtraData: map[string]string{- "account_name": "teststoragebytruffle",- },- }- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{"myAccountKey12345=="},},{- name: "found secret with invalid account name",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", validKeyInvalidAccountName)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_AzureStorage,- Verified: false,- ExtraData: map[string]string{- "account_name": "invalid",- },- },- },- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}