Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
added and updated pattern tests for detectors starting from p to q (#3710)
Nabeel Alam35943b41905e
diff --git a/pkg/detectors/packagecloud/packagecloud_integration_test.go b/pkg/detectors/packagecloud/packagecloud_integration_test.gonew file mode 100644index 000000000..ca8dc5c79--- /dev/null+++ b/pkg/detectors/packagecloud/packagecloud_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package packagecloud++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPackageCloud_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PACKAGECLOUD")+ inactiveSecret := testSecrets.MustGetField("PACKAGECLOUD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PackageCloud,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PackageCloud,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PackageCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PackageCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/packagecloud/packagecloud_test.go b/pkg/detectors/packagecloud/packagecloud_test.goindex ca8dc5c79..a088ba827 100644--- a/pkg/detectors/packagecloud/packagecloud_test.go+++ b/pkg/detectors/packagecloud/packagecloud_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package packagecloudimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPackageCloud_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PACKAGECLOUD")- inactiveSecret := testSecrets.MustGetField("PACKAGECLOUD_INACTIVE")+var (+ validPattern = "3a4c163d0c145e7346b53f6d9be8e0a18058a5b7f03e2b41"+ invalidPattern = "3a4c163d0c145e7346b53f6d?be8e0a18058a5b7f03e2b41"+ keyword = "packagecloud"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPackageCloud_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PackageCloud,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword packagecloud",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PackageCloud,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PackageCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PackageCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.gonew file mode 100644index 000000000..f3a1ff55c--- /dev/null+++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go@@ -0,0 +1,157 @@+//go:build detectors+// +build detectors++package pagerdutyapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPagerDutyApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAGERDUTYAPIKEY_TOKEN")+ invalidSecret := testSecrets.MustGetField("PAGERDUTYAPIKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within but not valid", invalidSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PagerDutyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Errorf("PagerDutyApiKey.FromData() verificationError = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr)+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("PagerDutyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.goindex f3a1ff55c..bc00c25b0 100644--- a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go+++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go@@ -1,156 +1,90 @@-//go:build detectors-// +build detectors-package pagerdutyapikeyimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPagerDutyApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAGERDUTYAPIKEY_TOKEN")- invalidSecret := testSecrets.MustGetField("PAGERDUTYAPIKEY_INACTIVE")+var (+ validPattern = "u+C4lGv0tct7TlvSUDoc"+ invalidPattern = "u+C4lGv0?ct7TlvSUDoc"+ keyword = "pagerdutyapikey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPagerDutyApiKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pagerdutyapikey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within but not valid", invalidSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PagerDutyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Errorf("PagerDutyApiKey.FromData() verificationError = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr)+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("PagerDutyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/pandadoc/pandadoc_integration_test.go b/pkg/detectors/pandadoc/pandadoc_integration_test.gonew file mode 100644index 000000000..52f140a88--- /dev/null+++ b/pkg/detectors/pandadoc/pandadoc_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pandadoc++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPandadoc_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PANDADOC")+ inactiveSecret := testSecrets.MustGetField("PANDADOC_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pandadoc,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pandadoc,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pandadoc.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pandadoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pandadoc/pandadoc_test.go b/pkg/detectors/pandadoc/pandadoc_test.goindex 52f140a88..d68ad4c8c 100644--- a/pkg/detectors/pandadoc/pandadoc_test.go+++ b/pkg/detectors/pandadoc/pandadoc_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pandadocimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPandadoc_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PANDADOC")- inactiveSecret := testSecrets.MustGetField("PANDADOC_INACTIVE")+var (+ validPattern = "gL1LKFp8p0q7ORUybiyovqIuGUsGjx4adxMRcHPh"+ invalidPattern = "gL1LKFp8p0q7ORUybiyo?qIuGUsGjx4adxMRcHPh"+ keyword = "pandadoc"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPandadoc_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pandadoc,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pandadoc",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pandadoc,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pandadoc.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pandadoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pandascore/pandascore_integration_test.go b/pkg/detectors/pandascore/pandascore_integration_test.gonew file mode 100644index 000000000..b6d5d005b--- /dev/null+++ b/pkg/detectors/pandascore/pandascore_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package pandascore++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPandaScore_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PANDASCORE")+ inactiveSecret := testSecrets.MustGetField("PANDASCORE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pandascore secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PandaScore,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pandascore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PandaScore,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PandaScore.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PandaScore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/pandascore/pandascore_test.go b/pkg/detectors/pandascore/pandascore_test.goindex b6d5d005b..46922fd00 100644--- a/pkg/detectors/pandascore/pandascore_test.go+++ b/pkg/detectors/pandascore/pandascore_test.go@@ -1,116 +1,90 @@-//go:build detectors-// +build detectors-package pandascoreimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPandaScore_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PANDASCORE")- inactiveSecret := testSecrets.MustGetField("PANDASCORE_INACTIVE")+var (+ validPattern = "iahNbqgnxq7tzca9hNBWiVexHeePoQ2nnXwN06wmaFSY5BMGyfq"+ invalidPattern = "iahNbqgnxq7tzca9hNBWiV?xHeePoQ2nnXwN06wmaFSY5BMGyfq"+ keyword = "pandascore"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPandaScore_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pandascore secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PandaScore,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pandascore",+ input: fmt.Sprintf("%s token = ' %s '", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pandascore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PandaScore,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = ' %s ' | ' %s '", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = ' %s '", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = ' %s '", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PandaScore.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PandaScore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/paperform/paperform_integration_test.go b/pkg/detectors/paperform/paperform_integration_test.gonew file mode 100644index 000000000..2b3de57e2--- /dev/null+++ b/pkg/detectors/paperform/paperform_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package paperform++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPaperform_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAPERFORM")+ inactiveSecret := testSecrets.MustGetField("PAPERFORM_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paperform secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paperform,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paperform secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paperform,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paperform.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paperform.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paperform/paperform_test.go b/pkg/detectors/paperform/paperform_test.goindex 2b3de57e2..23fc08e03 100644--- a/pkg/detectors/paperform/paperform_test.go+++ b/pkg/detectors/paperform/paperform_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package paperformimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPaperform_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAPERFORM")- inactiveSecret := testSecrets.MustGetField("PAPERFORM_INACTIVE")+var (+ validPattern = "NvBbE5vYVPhidc-tiZ7E6DvP.z4bAQj29KEZOCv_XEl1pIn.Y1q8JoeGNoSelGj.db1iQpLM0fsype86LD.Vk.p6yweF5A2MlfxXDrEd7nz2SBExnTpp4QRN94pBAeulPLLtzqTz--y_UuF1g6cjE_.kuW_u5He0QkBdLMkyAgSx94N3Csj9LY37XOmUp9IIi9LXpTvZGa8oywp6JuMfhzwg5OCEvdp9mx.UyfQcnnJtYzzP5dItmwfEC-KJIvjvS8LF2NU2w2japQHtnAJqBAn3_EP-FN78wHnDEWANANT0cfor6kDqyKraO0Y-26PdB6xBjm3_VpU.8hnKIyoKdLQ6S.HZwr5rx0Bx76zXTCBv4uEzhtDFcDqVPN8ZG_kE90P..ldReG0jU4w3YA2jbaOgi6i-8llYWGoCiFFBm3Od-zLOEDYL2BlGsUFRUkiEMjytCVDqcIOdfPT7GQcd3wdmort6FFv8SbCu95f2gBCcM.5.ZmMxIOybubMGmiRunYM8-pSaVvXfBQSkM2Eygh15tkKCDHf8X3InAkPh7HQn13mP5y1gFRLsVAUWb-91PeHASP6hluUEdsX3uLQ9OJFenKrk.0zS9Goy08bfttd4h4Jtb2JV8vbJ8-3Wb4AJWqf0eUALMxOChB3sSBKW37s4vDb1NKOnoqOeoYQUBijqRGu9YLKIAimwo7Uvl0CuD7bWNrERweBqNVWjfGhlE8Yvvklm5YhCk5XY02pOa3IjMf_TDKhbTr8bh_20SXevnDk80XKg_3mWbhuieL23kx835AokAg9JpEkgydBBqo8nzQg23R5xJzKRT64kgb5GBlzuM9Oxh7pXsmzlYf"+ invalidPattern = "NvBbE5vYVPhidc?tiZ7E6DvP.z4bAQj29KEZOCv_XEl1pIn.Y1q8JoeGNoSelGj.db1iQpLM0fsype86LD.Vk.p6yweF5A2MlfxXDrEd7nz2SBExnTpp4QRN94pBAeulPLLtzqTz--y_UuF1g6cjE_.kuW_u5He0QkBdLMkyAgSx94N3Csj9LY37XOmUp9IIi9LXpTvZGa8oywp6JuMfhzwg5OCEvdp9mx.UyfQcnnJtYzzP5dItmwfEC-KJIvjvS8LF2NU2w2japQHtnAJqBAn3_EP-FN78wHnDEWANANT0cfor6kDqyKraO0Y-26PdB6xBjm3_VpU.8hnKIyoKdLQ6S.HZwr5rx0Bx76zXTCBv4uEzhtDFcDqVPN8ZG_kE90P..ldReG0jU4w3YA2jbaOgi6i-8llYWGoCiFFBm3Od-zLOEDYL2BlGsUFRUkiEMjytCVDqcIOdfPT7GQcd3wdmort6FFv8SbCu95f2gBCcM.5.ZmMxIOybubMGmiRunYM8-pSaVvXfBQSkM2Eygh15tkKCDHf8X3InAkPh7HQn13mP5y1gFRLsVAUWb-91PeHASP6hluUEdsX3uLQ9OJFenKrk.0zS9Goy08bfttd4h4Jtb2JV8vbJ8-3Wb4AJWqf0eUALMxOChB3sSBKW37s4vDb1NKOnoqOeoYQUBijqRGu9YLKIAimwo7Uvl0CuD7bWNrERweBqNVWjfGhlE8Yvvklm5YhCk5XY02pOa3IjMf_TDKhbTr8bh_20SXevnDk80XKg_3mWbhuieL23kx835AokAg9JpEkgydBBqo8nzQg23R5xJzKRT64kgb5GBlzuM9Oxh7pXsmzlYf"+ keyword = "paperform"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPaperform_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paperform secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paperform,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword paperform",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paperform secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paperform,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Paperform.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Paperform.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/paralleldots/paralleldots_integration_test.go b/pkg/detectors/paralleldots/paralleldots_integration_test.gonew file mode 100644index 000000000..e5931de4d--- /dev/null+++ b/pkg/detectors/paralleldots/paralleldots_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package paralleldots++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestParalleldots_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PARALLELDOTS")+ inactiveSecret := testSecrets.MustGetField("PARALLELDOTS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ParallelDots,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ParallelDots,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paralleldots.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paralleldots.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paralleldots/paralleldots_test.go b/pkg/detectors/paralleldots/paralleldots_test.goindex e5931de4d..9fc8e7844 100644--- a/pkg/detectors/paralleldots/paralleldots_test.go+++ b/pkg/detectors/paralleldots/paralleldots_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package paralleldotsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestParalleldots_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PARALLELDOTS")- inactiveSecret := testSecrets.MustGetField("PARALLELDOTS_INACTIVE")+var (+ validPattern = "Ki82DlwEBI9dqr4k3cZPH6Z5fP39XgEPktFfgZTM4mW"+ invalidPattern = "Ki82DlwEBI9dqr4k3cZPH?Z5fP39XgEPktFfgZTM4mW"+ keyword = "paralleldots"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestParalleldots_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ParallelDots,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword paralleldots",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ParallelDots,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Paralleldots.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Paralleldots.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/parsehub/parsehub_integration_test.go b/pkg/detectors/parsehub/parsehub_integration_test.gonew file mode 100644index 000000000..0e3cd4560--- /dev/null+++ b/pkg/detectors/parsehub/parsehub_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package parsehub++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestParsehub_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PARSEHUB")+ inactiveSecret := testSecrets.MustGetField("PARSEHUB_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a parsehub secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Parsehub,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a parsehub secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Parsehub,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Parsehub.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Parsehub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/parsehub/parsehub_test.go b/pkg/detectors/parsehub/parsehub_test.goindex 0e3cd4560..79ad005ed 100644--- a/pkg/detectors/parsehub/parsehub_test.go+++ b/pkg/detectors/parsehub/parsehub_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package parsehubimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestParsehub_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PARSEHUB")- inactiveSecret := testSecrets.MustGetField("PARSEHUB_INACTIVE")+var (+ validPattern = "vZrEclFwOqeA"+ invalidPattern = "vZrEcl?wOqeA"+ keyword = "parsehub"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestParsehub_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a parsehub secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Parsehub,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword parsehub",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a parsehub secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Parsehub,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Parsehub.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Parsehub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/parsers/parsers_integration_test.go b/pkg/detectors/parsers/parsers_integration_test.gonew file mode 100644index 000000000..a396b9ef3--- /dev/null+++ b/pkg/detectors/parsers/parsers_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package parsers++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestParsers_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PARSERS")+ inactiveSecret := testSecrets.MustGetField("PARSERS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a parsers secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Parsers,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a parsers secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Parsers,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Parsers.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Parsers.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/parsers/parsers_test.go b/pkg/detectors/parsers/parsers_test.goindex a396b9ef3..eab1af259 100644--- a/pkg/detectors/parsers/parsers_test.go+++ b/pkg/detectors/parsers/parsers_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package parsersimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestParsers_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PARSERS")- inactiveSecret := testSecrets.MustGetField("PARSERS_INACTIVE")+var (+ validPattern = "3ui8k045w9b9n6c2fuqbr44p1fg64cjsvn2dv2uvkyxfmjler9ddsls6uqtizt7q"+ invalidPattern = "3ui8k045w9b9n6c2fuqbr44p1fg64cjs?n2dv2uvkyxfmjler9ddsls6uqtizt7q"+ keyword = "parsers"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestParsers_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a parsers secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Parsers,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword parsers",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a parsers secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Parsers,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Parsers.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Parsers.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/partnerstack/partnerstack_integration_test.go b/pkg/detectors/partnerstack/partnerstack_integration_test.gonew file mode 100644index 000000000..e5bcfb777--- /dev/null+++ b/pkg/detectors/partnerstack/partnerstack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package partnerstack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPartnerstack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PARTNERSTACK")+ inactiveSecret := testSecrets.MustGetField("PARTNERSTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Partnerstack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Partnerstack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Partnerstack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Partnerstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/partnerstack/partnerstack_test.go b/pkg/detectors/partnerstack/partnerstack_test.goindex e5bcfb777..aea076ae1 100644--- a/pkg/detectors/partnerstack/partnerstack_test.go+++ b/pkg/detectors/partnerstack/partnerstack_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package partnerstackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPartnerstack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PARTNERSTACK")- inactiveSecret := testSecrets.MustGetField("PARTNERSTACK_INACTIVE")+var (+ validPattern = "mDrW4HFb6fVh2ii74V41bh6EH52sWirlQuKr3svxfgGueyj32HW7OhIhNLZSgicW"+ invalidPattern = "mDrW4HFb6fVh2ii74V41bh6EH52sWirl?uKr3svxfgGueyj32HW7OhIhNLZSgicW"+ keyword = "partnerstack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPartnerstack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Partnerstack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword partnerstack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Partnerstack,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Partnerstack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Partnerstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pastebin/pastebin_integration_test.go b/pkg/detectors/pastebin/pastebin_integration_test.gonew file mode 100644index 000000000..e7aca2670--- /dev/null+++ b/pkg/detectors/pastebin/pastebin_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pastebin++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPastebin_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PASTEBIN")+ inactiveSecret := testSecrets.MustGetField("PASTEBIN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pastebin secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pastebin,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pastebin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pastebin,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pastebin.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pastebin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pastebin/pastebin_test.go b/pkg/detectors/pastebin/pastebin_test.goindex e7aca2670..5c1006ef9 100644--- a/pkg/detectors/pastebin/pastebin_test.go+++ b/pkg/detectors/pastebin/pastebin_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pastebinimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPastebin_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PASTEBIN")- inactiveSecret := testSecrets.MustGetField("PASTEBIN_INACTIVE")+var (+ validPattern = "92q9vNtJQRsjQDdbiYRRHBUNBYHSymKL"+ invalidPattern = "92q9vNtJQRsjQDdb?YRRHBUNBYHSymKL"+ keyword = "pastebin"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPastebin_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pastebin secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pastebin,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pastebin",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pastebin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pastebin,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pastebin.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pastebin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/paydirtapp/paydirtapp_integration_test.go b/pkg/detectors/paydirtapp/paydirtapp_integration_test.gonew file mode 100644index 000000000..950cab1a1--- /dev/null+++ b/pkg/detectors/paydirtapp/paydirtapp_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package paydirtapp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPaydirtyapp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAYDIRTAPP")+ inactiveSecret := testSecrets.MustGetField("PAYDIRTAPP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paydirtapp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paydirtapp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paydirtyapp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paydirtyapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paydirtapp/paydirtapp_test.go b/pkg/detectors/paydirtapp/paydirtapp_test.goindex 950cab1a1..ef44c0e44 100644--- a/pkg/detectors/paydirtapp/paydirtapp_test.go+++ b/pkg/detectors/paydirtapp/paydirtapp_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package paydirtappimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPaydirtyapp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAYDIRTAPP")- inactiveSecret := testSecrets.MustGetField("PAYDIRTAPP_INACTIVE")+var (+ validPattern = "teboslrqf1lirq5gx0ca6544y3tkzsq0"+ invalidPattern = "teboslrqf1lirq5g?0ca6544y3tkzsq0"+ keyword = "paydirtapp"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPaydirtyapp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paydirtapp,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword paydirtapp",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paydirtapp,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Paydirtyapp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Paydirtyapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/paymoapp/paymoapp_integration_test.go b/pkg/detectors/paymoapp/paymoapp_integration_test.gonew file mode 100644index 000000000..371bda62b--- /dev/null+++ b/pkg/detectors/paymoapp/paymoapp_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package paymoapp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPaymoapp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAYMOAPP")+ inactiveSecret := testSecrets.MustGetField("PAYMOAPP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paymoapp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paymoapp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paymoapp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paymoapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paymoapp/paymoapp_test.go b/pkg/detectors/paymoapp/paymoapp_test.goindex 371bda62b..6886a656f 100644--- a/pkg/detectors/paymoapp/paymoapp_test.go+++ b/pkg/detectors/paymoapp/paymoapp_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package paymoappimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPaymoapp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAYMOAPP")- inactiveSecret := testSecrets.MustGetField("PAYMOAPP_INACTIVE")+var (+ validPattern = "EpKoocVnygMTgD6XB0c4QQ2mYIqXlt1efBuIvuihzKjU"+ invalidPattern = "EpKoocVnygMTgD6XB0c4QQ?mYIqXlt1efBuIvuihzKjU"+ keyword = "paymoapp"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPaymoapp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paymoapp,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword paymoapp",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paymoapp,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Paymoapp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Paymoapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/paymongo/paymongo_integration_test.go b/pkg/detectors/paymongo/paymongo_integration_test.gonew file mode 100644index 000000000..cd7daf81e--- /dev/null+++ b/pkg/detectors/paymongo/paymongo_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package paymongo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPaymongo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAYMONGO")+ inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paymongo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paymongo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paymongo/paymongo_test.go b/pkg/detectors/paymongo/paymongo_test.goindex cd7daf81e..864918d4d 100644--- a/pkg/detectors/paymongo/paymongo_test.go+++ b/pkg/detectors/paymongo/paymongo_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package paymongoimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPaymongo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAYMONGO")- inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")+var (+ validPattern = "okrrdIMzwSl350mcZkrtMkzEJ_xuLaJc"+ invalidPattern = "okrrdIMzwSl350mc?krtMkzEJ_xuLaJc"+ keyword = "paymongo"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPaymongo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paymongo,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword paymongo",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paymongo,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/paypaloauth/paypaloauth_integration_test.go b/pkg/detectors/paypaloauth/paypaloauth_integration_test.gonew file mode 100644index 000000000..f2c8e0263--- /dev/null+++ b/pkg/detectors/paypaloauth/paypaloauth_integration_test.go@@ -0,0 +1,152 @@+//go:build detectors+// +build detectors++package paypaloauth++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPaypalOauth_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAYPALOAUTH_SECRET")+ inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")+ id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")++ newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")+ newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PaypalOauth,+ Verified: true,+ },+ {+ DetectorType: detectorspb.DetectorType_PaypalOauth,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PaypalOauth,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_PaypalOauth,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "new format, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PaypalOauth,+ Verified: false,+ },+ {+ DetectorType: detectorspb.DetectorType_PaypalOauth,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PaypalOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PaypalOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paypaloauth/paypaloauth_test.go b/pkg/detectors/paypaloauth/paypaloauth_test.goindex f2c8e0263..18e52675b 100644--- a/pkg/detectors/paypaloauth/paypaloauth_test.go+++ b/pkg/detectors/paypaloauth/paypaloauth_test.go@@ -1,152 +1,83 @@-//go:build detectors-// +build detectors-package paypaloauthimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPaypalOauth_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAYPALOAUTH_SECRET")- inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")- id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")-- newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")- newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")+var (+ validId = "Dddpt-5MySAKlcPX07LKjhzbHTbf9m2Xv9OFSw0bTdrZ"+ invalidId = "Dddpt-5MySAKlcPX07LKjh?bHTbf9m2Xv9OFSw0bTdrZ"+ validKey = "PDMDWZ3BAwfXHoU.P.pJVUPIYlkJ2jVP-ns2icvSlWeb-_0Qa5"+ invalidKey = "PDMDWZ3BAwfXHoU.P.pJVUPIY?kJ2jVP-ns2icvSlWeb-_0Qa5"+ keyword = "paypaloauth"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPaypalOauth_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PaypalOauth,- Verified: true,- },- {- DetectorType: detectorspb.DetectorType_PaypalOauth,- Verified: false,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PaypalOauth,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_PaypalOauth,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword paypaloauth",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validKey),+ want: []string{validId, validKey},},{- name: "new format, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PaypalOauth,- Verified: false,- },- {- DetectorType: detectorspb.DetectorType_PaypalOauth,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidKey),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PaypalOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PaypalOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/paystack/paystack_integration_test.go b/pkg/detectors/paystack/paystack_integration_test.gonew file mode 100644index 000000000..19d5531ae--- /dev/null+++ b/pkg/detectors/paystack/paystack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package paystack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPaystack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PAYSTACK_TOKEN")+ inactiveSecret := testSecrets.MustGetField("PAYSTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paystack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paystack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a paystack secret %s within but unverified", inactiveSecret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Paystack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Paystack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Paystack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/paystack/paystack_test.go b/pkg/detectors/paystack/paystack_test.goindex 19d5531ae..41b42d1f0 100644--- a/pkg/detectors/paystack/paystack_test.go+++ b/pkg/detectors/paystack/paystack_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package paystackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPaystack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PAYSTACK_TOKEN")- inactiveSecret := testSecrets.MustGetField("PAYSTACK_INACTIVE")+var (+ validPattern = "sk_xigrvarm_cJHGpWQwCTHajG2A2o8eC8TQaQGZdoMVhgXUA9Lm"+ invalidPattern = "sk_xigrvarm_cJHGpWQwCTHajG?A2o8eC8TQaQGZdoMVhgXUA9Lm"+ keyword = "paystack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPaystack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paystack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paystack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword paystack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a paystack secret %s within but unverified", inactiveSecret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Paystack,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Paystack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Paystack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pdflayer/pdflayer_integration_test.go b/pkg/detectors/pdflayer/pdflayer_integration_test.gonew file mode 100644index 000000000..d803ff921--- /dev/null+++ b/pkg/detectors/pdflayer/pdflayer_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pdflayer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPdfLayer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PDFLAYER")+ inactiveSecret := testSecrets.MustGetField("PDFLAYER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PdfLayer,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PdfLayer,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PdfLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PdfLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pdflayer/pdflayer_test.go b/pkg/detectors/pdflayer/pdflayer_test.goindex d803ff921..7896435b1 100644--- a/pkg/detectors/pdflayer/pdflayer_test.go+++ b/pkg/detectors/pdflayer/pdflayer_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pdflayerimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPdfLayer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PDFLAYER")- inactiveSecret := testSecrets.MustGetField("PDFLAYER_INACTIVE")+var (+ validPattern = "wv8d7n0j02zxzrsbcs8cyk9yvqch9gvr"+ invalidPattern = "wv8d7n0j02zxzrsb?s8cyk9yvqch9gvr"+ keyword = "pdflayer"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPdfLayer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PdfLayer,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pdflayer",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PdfLayer,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PdfLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PdfLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pdfshift/pdfshift_integration_test.go b/pkg/detectors/pdfshift/pdfshift_integration_test.gonew file mode 100644index 000000000..bbddc1251--- /dev/null+++ b/pkg/detectors/pdfshift/pdfshift_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pdfshift++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPdfShift_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PDFSHIFT")+ inactiveSecret := testSecrets.MustGetField("PDFSHIFT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PdfShift,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PdfShift,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PdfShift.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PdfShift.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pdfshift/pdfshift_test.go b/pkg/detectors/pdfshift/pdfshift_test.goindex bbddc1251..f747d8d1b 100644--- a/pkg/detectors/pdfshift/pdfshift_test.go+++ b/pkg/detectors/pdfshift/pdfshift_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pdfshiftimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPdfShift_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PDFSHIFT")- inactiveSecret := testSecrets.MustGetField("PDFSHIFT_INACTIVE")+var (+ validPattern = "b03ed226557f474fda3f5a8fdd498f7c"+ invalidPattern = "b03e?226557f474fda3f5a8fdd498f7c"+ keyword = "pdfshift"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPdfShift_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PdfShift,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pdfshift",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PdfShift,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PdfShift.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PdfShift.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/peopledatalabs/peopledatalabs_integration_test.go b/pkg/detectors/peopledatalabs/peopledatalabs_integration_test.gonew file mode 100644index 000000000..c913f7f04--- /dev/null+++ b/pkg/detectors/peopledatalabs/peopledatalabs_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package peopledatalabs++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPeopleDataLabs_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PEOPLEDATALABS")+ inactiveSecret := testSecrets.MustGetField("PEOPLEDATALABS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PeopleDataLabs,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PeopleDataLabs,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PeopleDataLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PeopleDataLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/peopledatalabs/peopledatalabs_test.go b/pkg/detectors/peopledatalabs/peopledatalabs_test.goindex c913f7f04..25523d477 100644--- a/pkg/detectors/peopledatalabs/peopledatalabs_test.go+++ b/pkg/detectors/peopledatalabs/peopledatalabs_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package peopledatalabsimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPeopleDataLabs_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PEOPLEDATALABS")- inactiveSecret := testSecrets.MustGetField("PEOPLEDATALABS_INACTIVE")+var (+ validPattern = "tp8k005gi460y93zclyrifwjnij6492iiwignkeuccqjkrs1rqcw0lsuvjm39ij9"+ invalidPattern = "tp8k005g?460y93zclyrifwjnij6492iiwignkeuccqjkrs1rqcw0lsuvjm39ij9"+ keyword = "peopledatalabs"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPeopleDataLabs_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PeopleDataLabs,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword peopledatalabs",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PeopleDataLabs,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PeopleDataLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PeopleDataLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pepipost/pepipost_integration_test.go b/pkg/detectors/pepipost/pepipost_integration_test.gonew file mode 100644index 000000000..ced4dc843--- /dev/null+++ b/pkg/detectors/pepipost/pepipost_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pepipost++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPepipost_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PEPIPOST_TOKEN")+ inactiveSecret := testSecrets.MustGetField("PEPIPOST_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pepipost secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pepipost,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pepipost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pepipost,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pepipost.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pepipost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pepipost/pepipost_test.go b/pkg/detectors/pepipost/pepipost_test.goindex ced4dc843..0d27ee25b 100644--- a/pkg/detectors/pepipost/pepipost_test.go+++ b/pkg/detectors/pepipost/pepipost_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pepipostimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPepipost_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PEPIPOST_TOKEN")- inactiveSecret := testSecrets.MustGetField("PEPIPOST_INACTIVE")+var (+ validPattern = "zajmEjyCrhFdm1gQXoZD6r8WKLubC-B2"+ invalidPattern = "zajmEjyCrhFdm1gQ?oZD6r8WKLubC-B2"+ keyword = "pepipost"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPepipost_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pepipost secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pepipost,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pepipost",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pepipost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pepipost,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pepipost.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pepipost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/percy/percy_integration_test.go b/pkg/detectors/percy/percy_integration_test.gonew file mode 100644index 000000000..aad571e5b--- /dev/null+++ b/pkg/detectors/percy/percy_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package percy++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPercy_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PERCY_VERIFIED")+ inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Percy,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Percy,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/percy/percy_test.go b/pkg/detectors/percy/percy_test.goindex aad571e5b..8942cbedb 100644--- a/pkg/detectors/percy/percy_test.go+++ b/pkg/detectors/percy/percy_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package percyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPercy_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PERCY_VERIFIED")- inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")+var (+ validPattern = "84f2cfA002913e5afbe0a43d71e49ac9389Ab4f4f827bceAed69ec34f844ed22"+ invalidPattern = "84f2cfA002913?5afbe0a43d71e49ac9389Ab4f4f827bceAed69ec34f844ed22"+ keyword = "percy"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPercy_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Percy,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword percy",+ input: fmt.Sprintf("%s token = 'PERCY_TOKEN=%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Percy,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = 'PERCY_TOKEN=%s' | 'PERCY_TOKEN=%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = 'PERCY_TOKEN=%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = 'PERCY_TOKEN=%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pinata/pinata_integration_test.go b/pkg/detectors/pinata/pinata_integration_test.gonew file mode 100644index 000000000..9231cef77--- /dev/null+++ b/pkg/detectors/pinata/pinata_integration_test.go@@ -0,0 +1,121 @@+//go:build detectors+// +build detectors++package pinata++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPinata_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PINATA")+ key := testSecrets.MustGetField("PINATA_KEY")+ inactiveSecret := testSecrets.MustGetField("PINATA_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s", secret, key)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pinata,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pinata,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pinata.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pinata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pinata/pinata_test.go b/pkg/detectors/pinata/pinata_test.goindex 9231cef77..fb46c46d8 100644--- a/pkg/detectors/pinata/pinata_test.go+++ b/pkg/detectors/pinata/pinata_test.go@@ -1,121 +1,83 @@-//go:build detectors-// +build detectors-package pinataimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPinata_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PINATA")- key := testSecrets.MustGetField("PINATA_KEY")- inactiveSecret := testSecrets.MustGetField("PINATA_INACTIVE")+var (+ validKey = "xw4xxzwjzerq6rf3zvd8zwnlh0yq62g4f7l97xxlg4u1043zrx4ndtptkoqdn49e"+ invalidKey = "xw4xxzwjzerq6r?3zvd8zwnlh0yq62g4f7l97xxlg4u1043zrx4ndtptkoqdn49e"+ validId = "lnurl7e15mgdgv1hyalo"+ invalidId = "lnurl7e15m?dgv1hyalo"+ keyword = "pinata"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPinata_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s", secret, key)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pinata,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pinata",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pinata,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pinata.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pinata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pipedream/pipedream_integration_test.go b/pkg/detectors/pipedream/pipedream_integration_test.gonew file mode 100644index 000000000..202da5a57--- /dev/null+++ b/pkg/detectors/pipedream/pipedream_integration_test.go@@ -0,0 +1,117 @@+//go:build detectors+// +build detectors++package pipedream++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPipedream_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PIPEDREAM")+ inactiveSecret := testSecrets.MustGetField("PIPEDREAM_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pipedream secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pipedream,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pipedream secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pipedream,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pipedream.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pipedream.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/pipedream/pipedream_test.go b/pkg/detectors/pipedream/pipedream_test.goindex 202da5a57..d6eeab70d 100644--- a/pkg/detectors/pipedream/pipedream_test.go+++ b/pkg/detectors/pipedream/pipedream_test.go@@ -1,116 +1,90 @@-//go:build detectors-// +build detectors-package pipedreamimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPipedream_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PIPEDREAM")- inactiveSecret := testSecrets.MustGetField("PIPEDREAM_INACTIVE")+var (+ validPattern = "dh6zkbdzrjlsz5gy4gj2zg9stqttbd65"+ invalidPattern = "dh6zkbdzrjlsz5gy?gj2zg9stqttbd65"+ keyword = "pipedream"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPipedream_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pipedream secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pipedream,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pipedream",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pipedream secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pipedream,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pipedream.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pipedream.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/pipedrive/pipedrive_integration_test.go b/pkg/detectors/pipedrive/pipedrive_integration_test.gonew file mode 100644index 000000000..271a4a3f0--- /dev/null+++ b/pkg/detectors/pipedrive/pipedrive_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pipedrive++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPipedrive_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PIPEDRIVE_TOKEN")+ inactiveSecret := testSecrets.MustGetField("PIPEDRIVE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pipedrive,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pipedrive,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pipedrive.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pipedrive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pipedrive/pipedrive_test.go b/pkg/detectors/pipedrive/pipedrive_test.goindex 271a4a3f0..25301e8c8 100644--- a/pkg/detectors/pipedrive/pipedrive_test.go+++ b/pkg/detectors/pipedrive/pipedrive_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pipedriveimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPipedrive_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PIPEDRIVE_TOKEN")- inactiveSecret := testSecrets.MustGetField("PIPEDRIVE_INACTIVE")+var (+ validPattern = "ZVt2kcilMoT5gvZDppcXkF5JPzyinJrkSwroZ9dB"+ invalidPattern = "ZVt2?cilMoT5gvZDppcXkF5JPzyinJrkSwroZ9dB"+ keyword = "pipedrive"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPipedrive_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pipedrive,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pipedrive",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pipedrive,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pipedrive.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pipedrive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pivotaltracker/pivotaltracker_integration_test.go b/pkg/detectors/pivotaltracker/pivotaltracker_integration_test.gonew file mode 100644index 000000000..5e2809a56--- /dev/null+++ b/pkg/detectors/pivotaltracker/pivotaltracker_integration_test.go@@ -0,0 +1,119 @@+//go:build detectors+// +build detectors++package pivotaltracker++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPivotalTracker_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PIVOTALTRACKER")+ secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PivotalTracker,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PivotalTracker,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pivotaltracker/pivotaltracker_test.go b/pkg/detectors/pivotaltracker/pivotaltracker_test.goindex 5e2809a56..9962cd484 100644--- a/pkg/detectors/pivotaltracker/pivotaltracker_test.go+++ b/pkg/detectors/pivotaltracker/pivotaltracker_test.go@@ -1,119 +1,91 @@-//go:build detectors-// +build detectors-package pivotaltrackerimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPivotalTracker_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PIVOTALTRACKER")- secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")+var (+ validPattern = "hzwlc7d249nw76xwx372n09spjhjgidv"+ invalidPattern = "hzwlc7d249nw76?wx372n09spjhjgidv"+ keyword = "pivotaltracker"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPivotalTracker_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PivotalTracker,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pivotaltracker",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PivotalTracker,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pixabay/pixabay_integration_test.go b/pkg/detectors/pixabay/pixabay_integration_test.gonew file mode 100644index 000000000..62c962ec3--- /dev/null+++ b/pkg/detectors/pixabay/pixabay_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pixabay++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPixabay_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PIXABAY")+ inactiveSecret := testSecrets.MustGetField("PIXABAY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pixabay secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pixabay,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pixabay secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pixabay,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pixabay.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pixabay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pixabay/pixabay_test.go b/pkg/detectors/pixabay/pixabay_test.goindex 62c962ec3..72673940f 100644--- a/pkg/detectors/pixabay/pixabay_test.go+++ b/pkg/detectors/pixabay/pixabay_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pixabayimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPixabay_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PIXABAY")- inactiveSecret := testSecrets.MustGetField("PIXABAY_INACTIVE")+var (+ validPattern = "yz4gjwgew94zn73y9ds7hlwob6ytl70827"+ invalidPattern = "yz4gjwgew94zn73y9?s7hlwob6ytl70827"+ keyword = "pixabay"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPixabay_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pixabay secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pixabay,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pixabay",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pixabay secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pixabay,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pixabay.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pixabay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/plaidkey/plaidkey_integration_test.go b/pkg/detectors/plaidkey/plaidkey_integration_test.gonew file mode 100644index 000000000..624c22e2f--- /dev/null+++ b/pkg/detectors/plaidkey/plaidkey_integration_test.go@@ -0,0 +1,126 @@+//go:build detectors+// +build detectors++package plaidkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPlaidKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PLAIDKEY_SECRET")+ inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")+ id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")+ // env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production+ env := "development"++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within plaidkey %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlaidKey,+ Verified: true,+ ExtraData: map[string]string{+ "environment": fmt.Sprintf("https://%s.plaid.com", env),+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within but plaidkey %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlaidKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PlaidKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PlaidKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/plaidkey/plaidkey_test.go b/pkg/detectors/plaidkey/plaidkey_test.goindex 624c22e2f..73241d75a 100644--- a/pkg/detectors/plaidkey/plaidkey_test.go+++ b/pkg/detectors/plaidkey/plaidkey_test.go@@ -1,126 +1,83 @@-//go:build detectors-// +build detectors-package plaidkeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPlaidKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PLAIDKEY_SECRET")- inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")- id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")- // env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production- env := "development"+var (+ validKey = "3vl81ihtozf9im7kqz7ldp6kxbsd8y"+ invalidKey = "3vl81ihtozf9im7?qz7ldp6kxbsd8y"+ validId = "ic1mh5b49ycvmz2vgvlgxtb0"+ invalidId = "ic1?h5b49ycvmz2vgvlgxtb0"+ keyword = "plaid"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPlaidKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within plaidkey %s", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlaidKey,- Verified: true,- ExtraData: map[string]string{- "environment": fmt.Sprintf("https://%s.plaid.com", env),- },- },- },- wantErr: false,+ name: "valid pattern - with keyword plaid",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within but plaidkey %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlaidKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PlaidKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PlaidKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/planetscale/planetscale_integration_test.go b/pkg/detectors/planetscale/planetscale_integration_test.gonew file mode 100644index 000000000..315f8f97b--- /dev/null+++ b/pkg/detectors/planetscale/planetscale_integration_test.go@@ -0,0 +1,164 @@+//go:build detectors+// +build detectors++package planetscale++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPlanetscale_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PLANET_SCALE_TOKEN")+ secretID := testSecrets.MustGetField("PLANET_SCALE_ID")+ inactiveSecret := testSecrets.MustGetField("PLANET_SCALE_TOKEN_INACTIVE")+ inactiveSecretID := testSecrets.MustGetField("PLANET_SCALE_ID_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanetScale,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s but not valid", inactiveSecret, inactiveSecretID)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanetScale,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanetScale,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanetScale,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Planetscale.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Planetscale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/planetscale/planetscale_test.go b/pkg/detectors/planetscale/planetscale_test.goindex 315f8f97b..da30c0d85 100644--- a/pkg/detectors/planetscale/planetscale_test.go+++ b/pkg/detectors/planetscale/planetscale_test.go@@ -1,163 +1,82 @@-//go:build detectors-// +build detectors-package planetscaleimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPlanetscale_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PLANET_SCALE_TOKEN")- secretID := testSecrets.MustGetField("PLANET_SCALE_ID")- inactiveSecret := testSecrets.MustGetField("PLANET_SCALE_TOKEN_INACTIVE")- inactiveSecretID := testSecrets.MustGetField("PLANET_SCALE_ID_INACTIVE")+var (+ validUsername = "fo57eya1lvvh"+ invalidUsername = "fo57ey?1lvvh"+ validPassword = "pscale_tkn_toLyJg1LfgY8vDQr9vhfF2HcT410U3q1alxt7012kcV"+ invalidPassword = "pscale_tkn_toLyJg1LfgY8vDQr?vhfF2HcT410U3q1alxt7012kcV"+ keyword = "planetscale"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPlanetscale_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanetScale,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s but not valid", inactiveSecret, inactiveSecretID)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanetScale,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword planetscale",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validPassword),+ want: []string{validUsername + ":" + validPassword},},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanetScale,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanetScale,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidPassword),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Planetscale.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Planetscale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/planetscaledb/planetscaledb_integration_test.go b/pkg/detectors/planetscaledb/planetscaledb_integration_test.gonew file mode 100644index 000000000..57a1ae3c4--- /dev/null+++ b/pkg/detectors/planetscaledb/planetscaledb_integration_test.go@@ -0,0 +1,129 @@+//go:build detectors+// +build detectors++package planetscaledb++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPlanetscaledb_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ username := testSecrets.MustGetField("PLANET_SCALEDB_USERNAME")+ host := testSecrets.MustGetField("PLANET_SCALEDB_HOST")+ password := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD")+ inactivePassword := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, password, host)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanetScaleDb,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, inactivePassword, host)),+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanetScaleDb,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Planetscaledb.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Planetscaledb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/planetscaledb/planetscaledb_test.go b/pkg/detectors/planetscaledb/planetscaledb_test.goindex 57a1ae3c4..a1a0fc223 100644--- a/pkg/detectors/planetscaledb/planetscaledb_test.go+++ b/pkg/detectors/planetscaledb/planetscaledb_test.go@@ -1,128 +1,84 @@-//go:build detectors-// +build detectors-package planetscaledbimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPlanetscaledb_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- username := testSecrets.MustGetField("PLANET_SCALEDB_USERNAME")- host := testSecrets.MustGetField("PLANET_SCALEDB_HOST")- password := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD")- inactivePassword := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD_INACTIVE")+var (+ validUsername = "82mygyuh2y23aw1k8lzv"+ invalidUsername = "8?mygyuh2y23aw1k8lzv"+ validPassword = "pscale_pw_iAhKQKjU8nUrHagaygAubhM6x0LTaBz6kOyqx9AIS6V"+ invalidPassword = "pscale_pw_iAhKQKjU8nUrHaga?gAubhM6x0LTaBz6kOyqx9AIS6V"+ validHost = "gcp.connect.psdb.cloud"+ invalidHost = "gcp?connect.psdb.cloud"+ keyword = "planetscaledb"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPlanetscaledb_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, password, host)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanetScaleDb,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword planetscaledb",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validPassword, keyword, validHost),+ want: []string{validHost + "\t" + validUsername + "\t" + validPassword},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, inactivePassword, host)),- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanetScaleDb,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidPassword, keyword, invalidHost),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Planetscaledb.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Planetscaledb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/planviewleankit/planviewleankit_integration_test.go b/pkg/detectors/planviewleankit/planviewleankit_integration_test.gonew file mode 100644index 000000000..99a39767f--- /dev/null+++ b/pkg/detectors/planviewleankit/planviewleankit_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package planviewleankit++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPlanviewLeanKit_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PLANVIEWLEANKIT")+ inactiveSecret := testSecrets.MustGetField("PLANVIEWLEANKIT_INACTIVE")+ subdomain := testSecrets.MustGetField("PLANVIEWLEANKIT_SUBDOMAIN")++ // log.Println(secret)+ // log.Println(inactiveSecret)+ // log.Println(subdomain)++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within", subdomain, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanviewLeanKit,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within but not valid", subdomain, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PlanviewLeanKit,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PlanviewLeanKit.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PlanviewLeanKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ s.FromData(ctx, false, data)+ }+ })+ }+}diff --git a/pkg/detectors/planviewleankit/planviewleankit_test.go b/pkg/detectors/planviewleankit/planviewleankit_test.goindex 99a39767f..bb6cb22d9 100644--- a/pkg/detectors/planviewleankit/planviewleankit_test.go+++ b/pkg/detectors/planviewleankit/planviewleankit_test.go@@ -1,121 +1,82 @@-//go:build detectors-// +build detectors-package planviewleankitimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPlanviewLeanKit_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PLANVIEWLEANKIT")- inactiveSecret := testSecrets.MustGetField("PLANVIEWLEANKIT_INACTIVE")- subdomain := testSecrets.MustGetField("PLANVIEWLEANKIT_SUBDOMAIN")-- // log.Println(secret)- // log.Println(inactiveSecret)- // log.Println(subdomain)+var (+ validKey = "b1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277bd5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4f"+ invalidKey = "B1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277bd5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4F"+ validSubDomain = "subdomain.pYwPPnHSc"+ invalidSubDomain = "?ubdomain.pYwPPnHS?"+ keyword = "planviewleankit"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPlanviewLeanKit_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within", subdomain, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanviewLeanKit,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword planviewleankit",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSubDomain),+ want: []string{validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within but not valid", subdomain, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PlanviewLeanKit,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSubDomain),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PlanviewLeanKit.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PlanviewLeanKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/planyo/planyo_integration_test.go b/pkg/detectors/planyo/planyo_integration_test.gonew file mode 100644index 000000000..03730cd2f--- /dev/null+++ b/pkg/detectors/planyo/planyo_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package planyo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPlanyo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PLANYO")+ inactiveSecret := testSecrets.MustGetField("PLANYO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planyo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Planyo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a planyo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Planyo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Planyo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Planyo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/planyo/planyo_test.go b/pkg/detectors/planyo/planyo_test.goindex 03730cd2f..dc0bfc3f3 100644--- a/pkg/detectors/planyo/planyo_test.go+++ b/pkg/detectors/planyo/planyo_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package planyoimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPlanyo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PLANYO")- inactiveSecret := testSecrets.MustGetField("PLANYO_INACTIVE")+var (+ validPattern = "big57xmqvemmsopa2s7s3805oo70dzxk7subgcao8zerjg89ze8walz5si63x7"+ invalidPattern = "big57xmqvemmsopa2s7s3805oo70dzx?7subgcao8zerjg89ze8walz5si63x7"+ keyword = "planyo"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPlanyo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planyo secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Planyo,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword planyo",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a planyo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Planyo,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Planyo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Planyo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/plivo/plivo_integration_test.go b/pkg/detectors/plivo/plivo_integration_test.gonew file mode 100644index 000000000..26db005f4--- /dev/null+++ b/pkg/detectors/plivo/plivo_integration_test.go@@ -0,0 +1,123 @@+//go:build detectors+// +build detectors++package plivo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPlivo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PLIVO_TOKEN")+ inactiveSecret := testSecrets.MustGetField("PLIVO_INACTIVE")+ id := testSecrets.MustGetField("PLIVO_ID")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Plivo,+ Redacted: id,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/ but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Plivo,+ Redacted: id,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Plivo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Plivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/plivo/plivo_test.go b/pkg/detectors/plivo/plivo_test.goindex 26db005f4..7b11b9212 100644--- a/pkg/detectors/plivo/plivo_test.go+++ b/pkg/detectors/plivo/plivo_test.go@@ -1,123 +1,83 @@-//go:build detectors-// +build detectors-package plivoimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPlivo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PLIVO_TOKEN")- inactiveSecret := testSecrets.MustGetField("PLIVO_INACTIVE")- id := testSecrets.MustGetField("PLIVO_ID")+var (+ validId = "YGIQXPGZSVVGVGOREMAE"+ invalidId = "YGIQXPGZS?VGVGOREMAE"+ validKey = "qFj32Da8vf-g_-8qLu9P_k8XPyAHGrvKrzGTQIN4"+ invalidKey = "qFj32Da8vf-g?-8qLu9P_k8XPyAHGrvKrzGTQIN4"+ keyword = "plivo"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPlivo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/", secret, id)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Plivo,- Redacted: id,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword plivo",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validKey),+ want: []string{validKey + validId},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/ but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Plivo,- Redacted: id,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidKey),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Plivo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Plivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/podio/podio_integration_test.go b/pkg/detectors/podio/podio_integration_test.gonew file mode 100644index 000000000..b7a162f03--- /dev/null+++ b/pkg/detectors/podio/podio_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package podio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPodio_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PODIO")+ inactiveSecret := testSecrets.MustGetField("PODIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a podio secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Podio,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a podio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Podio,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Podio.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Podio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/podio/podio_test.go b/pkg/detectors/podio/podio_test.goindex b7a162f03..d089c4c87 100644--- a/pkg/detectors/podio/podio_test.go+++ b/pkg/detectors/podio/podio_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package podioimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPodio_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PODIO")- inactiveSecret := testSecrets.MustGetField("PODIO_INACTIVE")+var (+ validPattern = "d3ij9dt7n9pu9enq0seldrpwcbl6p1ky"+ invalidPattern = "d3ij9dt7n9pu9enq?seldrpwcbl6p1ky"+ keyword = "podio"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPodio_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a podio secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Podio,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword podio",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a podio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Podio,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Podio.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Podio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pollsapi/pollsapi_integration_test.go b/pkg/detectors/pollsapi/pollsapi_integration_test.gonew file mode 100644index 000000000..7e28de28d--- /dev/null+++ b/pkg/detectors/pollsapi/pollsapi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pollsapi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPollsAPI_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POLLSAPI")+ inactiveSecret := testSecrets.MustGetField("POLLSAPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PollsAPI,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PollsAPI,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PollsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PollsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pollsapi/pollsapi_test.go b/pkg/detectors/pollsapi/pollsapi_test.goindex 7e28de28d..2c7dbdd20 100644--- a/pkg/detectors/pollsapi/pollsapi_test.go+++ b/pkg/detectors/pollsapi/pollsapi_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pollsapiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPollsAPI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POLLSAPI")- inactiveSecret := testSecrets.MustGetField("POLLSAPI_INACTIVE")+var (+ validPattern = "WHWPPKLOD23IRAUKKY24V9WWV6IC"+ invalidPattern = "WHWPPKLOD23IRA?KKY24V9WWV6IC"+ keyword = "pollsapi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPollsAPI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PollsAPI,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pollsapi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PollsAPI,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PollsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PollsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/poloniex/poloniex_integration_test.go b/pkg/detectors/poloniex/poloniex_integration_test.gonew file mode 100644index 000000000..22dd53d4a--- /dev/null+++ b/pkg/detectors/poloniex/poloniex_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package poloniex++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPoloniex_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ key := testSecrets.MustGetField("POLONIEX_KEY")+ inactiveKey := testSecrets.MustGetField("POLONIEX_KEY_INACTIVE")+ secret := testSecrets.MustGetField("POLONIEX")+ inactiveSecret := testSecrets.MustGetField("POLONIEX_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within", key, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Poloniex,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Poloniex,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Poloniex.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Poloniex.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/poloniex/poloniex_test.go b/pkg/detectors/poloniex/poloniex_test.goindex 22dd53d4a..47b38fe7e 100644--- a/pkg/detectors/poloniex/poloniex_test.go+++ b/pkg/detectors/poloniex/poloniex_test.go@@ -1,122 +1,83 @@-//go:build detectors-// +build detectors-package polonieximport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPoloniex_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- key := testSecrets.MustGetField("POLONIEX_KEY")- inactiveKey := testSecrets.MustGetField("POLONIEX_KEY_INACTIVE")- secret := testSecrets.MustGetField("POLONIEX")- inactiveSecret := testSecrets.MustGetField("POLONIEX_INACTIVE")+var (+ validKey = "2ELRAUEF-NB06CX0R-ZGXL88HF-BX8B77S6"+ invalidKey = "2ELRAUEF?NB06CX0R-ZGXL88HF-BX8B77S6"+ validSecret = "d5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4fb1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277b"+ invalidSecret = "D5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4fb1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277B"+ keyword = "poloniex"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPoloniex_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within", key, secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Poloniex,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword poloniex",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),+ want: []string{validKey + validSecret},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Poloniex,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Poloniex.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Poloniex.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/polygon/polygon_integration_test.go b/pkg/detectors/polygon/polygon_integration_test.gonew file mode 100644index 000000000..3565f38e4--- /dev/null+++ b/pkg/detectors/polygon/polygon_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package polygon++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPolygon_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POLYGON_TOKEN")+ inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Polygon,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Polygon,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/polygon/polygon_test.go b/pkg/detectors/polygon/polygon_test.goindex 3565f38e4..dda41ae02 100644--- a/pkg/detectors/polygon/polygon_test.go+++ b/pkg/detectors/polygon/polygon_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package polygonimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPolygon_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POLYGON_TOKEN")- inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")+var (+ validPattern = "vfINal1N4C0YH6fME3dxmRGaeuO1WjnP"+ invalidPattern = "vfINal1N4C0YH6fM?3dxmRGaeuO1WjnP"+ keyword = "polygon"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPolygon_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Polygon,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword polygon",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Polygon,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/portainer/portainer_integration_test.go b/pkg/detectors/portainer/portainer_integration_test.gonew file mode 100644index 000000000..49ca42d4a--- /dev/null+++ b/pkg/detectors/portainer/portainer_integration_test.go@@ -0,0 +1,131 @@+//go:build detectors+// +build detectors++package portainer++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPortainer_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PORTAINER")+ endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")+ inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Portainer,+ Verified: true,+ RawV2: []byte(secret + endpoint),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Portainer,+ Verified: false,+ RawV2: []byte(inactiveSecret + endpoint),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/portainer/portainer_test.go b/pkg/detectors/portainer/portainer_test.goindex 49ca42d4a..56cbe91f6 100644--- a/pkg/detectors/portainer/portainer_test.go+++ b/pkg/detectors/portainer/portainer_test.go@@ -1,130 +1,82 @@-//go:build detectors-// +build detectors-package portainerimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPortainer_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PORTAINER")- endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")- inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")+var (+ validEndpoint = "http://>xC'w//b7U@CtF|>|Fqw'2Z"+ invalidEndpoint = "?ttp://>xC'w//b7U@CtF|>|Fqw'2?"+ validToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.EHHJE6Aht7Exhje8rUMLScr2bxcoTvWBl9bjMYZhCMYMLPD3EpUZL9SNd839DcI95lYtMfclPffpFrrJ0BbgryxnrfUSeeSKHu.W9Ur5_DLIBpXO3mfh404_7Kt9o8XZRnLTLyam2fdhB_"+ invalidToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.EHHJE6Aht7Exhje8rUM?Scr2bxcoTvWBl9bjMYZhCMYMLPD3EpUZL9SNd839DcI95lYtMfclPffpFrrJ0BbgryxnrfUSeeSKHu.W9Ur5_DLIBpXO3mfh404_7Kt9o8XZRnLTLyam2fdhB_"+ keyword = "portainer"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPortainer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Portainer,- Verified: true,- RawV2: []byte(secret + endpoint),- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword portainer",+ input: fmt.Sprintf("%s token - '%s;'\n%s token - '%s'\n", keyword, validEndpoint, keyword, validToken),+ want: []string{validToken + validEndpoint},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Portainer,- Verified: false,- RawV2: []byte(inactiveSecret + endpoint),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s;'\n%s token - '%s'\n", keyword, invalidEndpoint, keyword, invalidToken),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.goindex 257f3b6c4..0f0c6fec3 100644--- a/pkg/detectors/portainertoken/portainertoken.go+++ b/pkg/detectors/portainertoken/portainertoken.go@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultu, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)if err != nil {+ fmt.Printf("\nINVALID URL\n")// if the URL is invalid just move onto the next onecontinue}diff --git a/pkg/detectors/portainertoken/portainertoken_integration_test.go b/pkg/detectors/portainertoken/portainertoken_integration_test.gonew file mode 100644index 000000000..cf2c35d22--- /dev/null+++ b/pkg/detectors/portainertoken/portainertoken_integration_test.go@@ -0,0 +1,131 @@+//go:build detectors+// +build detectors++package portainertoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPortainertoken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PORTAINERTOKEN")+ inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")+ endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PortainerToken,+ Verified: true,+ RawV2: []byte(secret + endpoint),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PortainerToken,+ Verified: false,+ RawV2: []byte(inactiveSecret + endpoint),+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/portainertoken/portainertoken_test.go b/pkg/detectors/portainertoken/portainertoken_test.goindex cf2c35d22..21b74bbef 100644--- a/pkg/detectors/portainertoken/portainertoken_test.go+++ b/pkg/detectors/portainertoken/portainertoken_test.go@@ -1,130 +1,82 @@-//go:build detectors-// +build detectors-package portainertokenimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPortainertoken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PORTAINERTOKEN")- inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")- endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")+var (+ validKey = "ptr_9zmQMKyeMqEB_pei887xQBZGhGWm1jXCIT0gI"+ invalidKey = "ptr_9zmQMKyeMqEB_pei?87xQBZGhGWm1jXCIT0gI"+ validEndpoint = "http://api.SAaiuc8123.com:12345"+ invalidEndpoint = "?ttp://api.SAaiuc8123.com:12345"+ keyword = "portainertoken"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPortainertoken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PortainerToken,- Verified: true,- RawV2: []byte(secret + endpoint),- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword portainertoken",+ input: fmt.Sprintf("%s token - '%s'\nportainer token - '%s'\n", keyword, validKey, validEndpoint),+ want: []string{validKey + validEndpoint},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PortainerToken,- Verified: false,- RawV2: []byte(inactiveSecret + endpoint),- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\nportainer token - '%s'\n", keyword, invalidKey, invalidEndpoint),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/positionstack/positionstack_integration_test.go b/pkg/detectors/positionstack/positionstack_integration_test.gonew file mode 100644index 000000000..f7494337b--- /dev/null+++ b/pkg/detectors/positionstack/positionstack_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package positionstack++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPositionStack_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POSITIONSTACK")+ inactiveSecret := testSecrets.MustGetField("POSITIONSTACK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a positionstack secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PositionStack,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a positionstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PositionStack,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PositionStack.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PositionStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/positionstack/positionstack_test.go b/pkg/detectors/positionstack/positionstack_test.goindex f7494337b..b7d6219c1 100644--- a/pkg/detectors/positionstack/positionstack_test.go+++ b/pkg/detectors/positionstack/positionstack_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package positionstackimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPositionStack_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POSITIONSTACK")- inactiveSecret := testSecrets.MustGetField("POSITIONSTACK_INACTIVE")+var (+ validPattern = "iUgwG0nYZt0TY1x5bfyWMJj02PhW7EGX"+ invalidPattern = "iUgwG0nYZt0TY1x5?fyWMJj02PhW7EGX"+ keyword = "positionstack"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPositionStack_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a positionstack secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PositionStack,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword positionstack",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a positionstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PositionStack,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PositionStack.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PositionStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/postageapp/postageapp_integration_test.go b/pkg/detectors/postageapp/postageapp_integration_test.gonew file mode 100644index 000000000..37e7596f7--- /dev/null+++ b/pkg/detectors/postageapp/postageapp_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package postageapp++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPostageApp_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POSTAGEAPP")+ inactiveSecret := testSecrets.MustGetField("POSTAGEAPP_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postageapp secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PostageApp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postageapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PostageApp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PostageApp.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PostageApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/postageapp/postageapp_test.go b/pkg/detectors/postageapp/postageapp_test.goindex 37e7596f7..a3f7dc482 100644--- a/pkg/detectors/postageapp/postageapp_test.go+++ b/pkg/detectors/postageapp/postageapp_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package postageappimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPostageApp_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POSTAGEAPP")- inactiveSecret := testSecrets.MustGetField("POSTAGEAPP_INACTIVE")+var (+ validPattern = "VJhmldVkyWB2bmLumsZzOtJKfxliCAXP"+ invalidPattern = "VJh?ldVkyWB2bmLumsZzOtJKfxliCAXP"+ keyword = "postageapp"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPostageApp_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postageapp secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PostageApp,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword postageapp",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postageapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PostageApp,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PostageApp.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PostageApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/postbacks/postbacks_integration_test.go b/pkg/detectors/postbacks/postbacks_integration_test.gonew file mode 100644index 000000000..fd032f3f1--- /dev/null+++ b/pkg/detectors/postbacks/postbacks_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package postbacks++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPostbacks_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POSTBACKS")+ inactiveSecret := testSecrets.MustGetField("POSTBACKS_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postbacks secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postbacks,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postbacks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postbacks,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Postbacks.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Postbacks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/postbacks/postbacks_test.go b/pkg/detectors/postbacks/postbacks_test.goindex fd032f3f1..d011b068e 100644--- a/pkg/detectors/postbacks/postbacks_test.go+++ b/pkg/detectors/postbacks/postbacks_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package postbacksimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPostbacks_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POSTBACKS")- inactiveSecret := testSecrets.MustGetField("POSTBACKS_INACTIVE")+var (+ validPattern = "64176f2e-e1da-3e29-2a08-19fa8bcb1838"+ invalidPattern = "64176f2e?e1da-3e29-2a08-19fa8bcb1838"+ keyword = "postbacks"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPostbacks_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postbacks secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postbacks,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword postbacks",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postbacks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postbacks,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Postbacks.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Postbacks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/postgres/postgres_integration_test.go b/pkg/detectors/postgres/postgres_integration_test.gonew file mode 100644index 000000000..107ec0b24--- /dev/null+++ b/pkg/detectors/postgres/postgres_integration_test.go@@ -0,0 +1,406 @@+//go:build detectors+// +build detectors++package postgres++import (+ "bytes"+ "context"+ "errors"+ "fmt"+ "os/exec"+ "strings"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/lib/pq"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++var postgresDockerHash string++const (+ postgresUser = "postgres"+ postgresPass = "23201da=b56ca236f3dc6736c0f9afad"+ postgresHost = "localhost"+ postgresPort = "5434" // Do not use 5433, as local dev environments can use it for other things++ inactivePass = "inactive"+ inactiveHost = "192.0.2.0"+)++func TestPostgres_FromChunk(t *testing.T) {+ if err := startPostgres(); err != nil {+ if exitErr, ok := err.(*exec.ExitError); ok {+ t.Fatalf("could not start local postgres: %v w/stderr:\n%s", err, string(exitErr.Stderr))+ } else {+ t.Fatalf("could not start local postgres: %v", err)+ }+ }+ defer stopPostgres()++ // The detector is written to connect to the database 'postgres' if no explicit database is found in the candidate+ // secret (because pq uses 'postgres' as a default if no database is specified). If the target cluster doesn't+ // actually have a database with this name, but our credentials are good, then Postgres will give us a "missing+ // database" error message instead of an authentication failure.+ //+ // Unfortunately, directly validating this in the automated tests is awkward because the docker image's POSTGRES_DB+ // environment variable doesn't appear to work: The database created is always named 'postgres', no matter what+ // POSTGRES_DB is set to. This means that we can't replicate a cluster that has no database named 'postgres', so we+ // can't directly test what happens if we see one. To work around this, all the automated tests try to connect to+ // the nonexistent database 'postgres2'. In this way, we test the logic of attempting to connect to a non-existent+ // database, even though the test cases are the inverse of what we'd see in the wild.+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "found connection URI with ssl mode unset, verified",+ s: Scanner{detectLoopback: true},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: true,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "<unset>"},+ },+ },+ wantErr: false,+ },+ {+ name: "found connection URI with ssl mode 'prefer', verified",+ s: Scanner{detectLoopback: true},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=prefer`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: true,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "prefer"},+ },+ },+ wantErr: false,+ },+ {+ name: "found connection URI with ssl mode 'allow', verified",+ s: Scanner{detectLoopback: true},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=allow`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: true,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "allow"},+ },+ },+ wantErr: false,+ },+ {+ name: "found connection URI with requiressl=0, verified",+ s: Scanner{detectLoopback: true},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=0`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: true,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "prefer"},+ },+ },+ wantErr: false,+ },+ {+ name: "found connection URI without database, verified",+ s: Scanner{detectLoopback: true},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: true,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "<unset>"},+ },+ },+ wantErr: false,+ },+ {+ name: "found connection URI, unverified",+ s: Scanner{detectLoopback: true},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, inactivePass, postgresHost, postgresPort)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: false,+ Raw: []byte("postgresql://postgres:inactive@localhost:5434"),+ RawV2: []byte("postgresql://postgres:inactive@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "<unset>"},+ },+ },+ wantErr: false,+ },+ {+ name: "ignored localhost",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "localhost", postgresPort)),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "ignored 127.0.0.1",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "127.0.0.1", postgresPort)),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ {+ name: "found connection URI, unverified due to error - inactive host",+ s: Scanner{},+ args: func() args {+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancel()+ return args{+ ctx: ctx,+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, inactiveHost, postgresPort)),+ verify: true,+ }+ }(),+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: false,+ Raw: []byte("postgresql://postgres:[email protected]:5434"),+ RawV2: []byte("postgresql://postgres:[email protected]:5434"),+ ExtraData: map[string]string{"sslmode": "<unset>"},+ }+ r.SetVerificationError(errors.New("i/o timeout"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found connection URI, unverified due to error - wrong port",+ s: Scanner{detectLoopback: true},+ args: func() args {+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancel()+ return args{+ ctx: ctx,+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s/postgres2`, postgresUser, postgresPass, postgresHost)),+ verify: true,+ }+ }(),+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: false,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),+ ExtraData: map[string]string{"sslmode": "<unset>"},+ }+ r.SetVerificationError(errors.New("connection refused"))+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found connection URI, unverified due to error - ssl not supported (using sslmode)",+ s: Scanner{detectLoopback: true},+ args: func() args {+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancel()+ return args{+ ctx: ctx,+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=require`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ }+ }(),+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: false,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "require"},+ }+ r.SetVerificationError(pq.ErrSSLNotSupported)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ {+ name: "found connection URI, unverified due to error - ssl not supported (using requiressl)",+ s: Scanner{detectLoopback: true},+ args: func() args {+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)+ defer cancel()+ return args{+ ctx: ctx,+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=1`, postgresUser, postgresPass, postgresHost, postgresPort)),+ verify: true,+ }+ }(),+ want: func() []detectors.Result {+ r := detectors.Result{+ DetectorType: detectorspb.DetectorType_Postgres,+ Verified: false,+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),+ ExtraData: map[string]string{"sslmode": "require"},+ }+ r.SetVerificationError(pq.ErrSSLNotSupported)+ return []detectors.Result{r}+ }(),+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ gotErr := ""+ if got[i].VerificationError() != nil {+ gotErr = got[i].VerificationError().Error()+ }+ wantErr := ""+ if tt.want[i].VerificationError() != nil {+ wantErr = tt.want[i].VerificationError().Error()+ }+ if gotErr != wantErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "AnalysisInfo")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func dockerLogLine(hash string, needle string) chan struct{} {+ ch := make(chan struct{}, 1)+ go func() {+ for {+ out, err := exec.Command("docker", "logs", hash).CombinedOutput()+ if err != nil {+ panic(err)+ }+ if strings.Contains(string(out), needle) {+ ch <- struct{}{}+ return+ }+ time.Sleep(1 * time.Second)+ }+ }()+ return ch+}++func startPostgres() error {+ cmd := exec.Command(+ "docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,+ "-e", "POSTGRES_PASSWORD="+postgresPass,+ "-e", "POSTGRES_USER="+postgresUser,+ "-d", "postgres",+ )+ fmt.Println(cmd.String())+ out, err := cmd.Output()+ if err != nil {+ return err+ }+ postgresDockerHash = string(bytes.TrimSpace(out))+ select {+ case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):+ return nil+ case <-time.After(30 * time.Second):+ stopPostgres()+ return errors.New("timeout waiting for postgres database to be ready")+ }+}++func stopPostgres() {+ exec.Command("docker", "kill", postgresDockerHash).Run()+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.goindex 107ec0b24..0570cca9a 100644--- a/pkg/detectors/postgres/postgres_test.go+++ b/pkg/detectors/postgres/postgres_test.go@@ -1,406 +1,83 @@-//go:build detectors-// +build detectors-package postgresimport (- "bytes""context"- "errors""fmt"- "os/exec"- "strings""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/lib/pq"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-var postgresDockerHash string--const (- postgresUser = "postgres"- postgresPass = "23201da=b56ca236f3dc6736c0f9afad"- postgresHost = "localhost"- postgresPort = "5434" // Do not use 5433, as local dev environments can use it for other things-- inactivePass = "inactive"- inactiveHost = "192.0.2.0"+var (+ validUriPattern = "postgres://sN19x:[email protected]:5432"+ invalidUriPattern = "?ostgres://sN19x:[email protected]:5432"+ validConnStrPartPattern = "gVmMTdkwLwmZljcIOXhEmuZ='.jD#=-;|9tD!r^6('"+ invalidConnStrPartPattern = "gVmMTdkwLwmZljcIOXhEmu?='.jD#=-;|9tD!r^6('"+ keyword = "postgres")-func TestPostgres_FromChunk(t *testing.T) {- if err := startPostgres(); err != nil {- if exitErr, ok := err.(*exec.ExitError); ok {- t.Fatalf("could not start local postgres: %v w/stderr:\n%s", err, string(exitErr.Stderr))- } else {- t.Fatalf("could not start local postgres: %v", err)- }- }- defer stopPostgres()-- // The detector is written to connect to the database 'postgres' if no explicit database is found in the candidate- // secret (because pq uses 'postgres' as a default if no database is specified). If the target cluster doesn't- // actually have a database with this name, but our credentials are good, then Postgres will give us a "missing- // database" error message instead of an authentication failure.- //- // Unfortunately, directly validating this in the automated tests is awkward because the docker image's POSTGRES_DB- // environment variable doesn't appear to work: The database created is always named 'postgres', no matter what- // POSTGRES_DB is set to. This means that we can't replicate a cluster that has no database named 'postgres', so we- // can't directly test what happens if we see one. To work around this, all the automated tests try to connect to- // the nonexistent database 'postgres2'. In this way, we test the logic of attempting to connect to a non-existent- // database, even though the test cases are the inverse of what we'd see in the wild.- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPostgres_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- {- name: "found connection URI with ssl mode unset, verified",- s: Scanner{detectLoopback: true},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postgres,- Verified: true,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "<unset>"},- },- },- wantErr: false,- },- {- name: "found connection URI with ssl mode 'prefer', verified",- s: Scanner{detectLoopback: true},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=prefer`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postgres,- Verified: true,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "prefer"},- },- },- wantErr: false,- },- {- name: "found connection URI with ssl mode 'allow', verified",- s: Scanner{detectLoopback: true},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=allow`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postgres,- Verified: true,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "allow"},- },- },- wantErr: false,- },- {- name: "found connection URI with requiressl=0, verified",- s: Scanner{detectLoopback: true},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=0`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postgres,- Verified: true,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "prefer"},- },- },- wantErr: false,+ name: "valid pattern - with keyword postgres",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validUriPattern, keyword, validConnStrPartPattern),+ want: []string{validUriPattern},},{- name: "found connection URI without database, verified",- s: Scanner{detectLoopback: true},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postgres,- Verified: true,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "<unset>"},- },- },- wantErr: false,- },- {- name: "found connection URI, unverified",- s: Scanner{detectLoopback: true},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, inactivePass, postgresHost, postgresPort)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postgres,- Verified: false,- Raw: []byte("postgresql://postgres:inactive@localhost:5434"),- RawV2: []byte("postgresql://postgres:inactive@localhost:5434"),- ExtraData: map[string]string{"sslmode": "<unset>"},- },- },- wantErr: false,- },- {- name: "ignored localhost",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "localhost", postgresPort)),- verify: true,- },- want: nil,- wantErr: false,- },- {- name: "ignored 127.0.0.1",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "127.0.0.1", postgresPort)),- verify: true,- },- want: nil,- wantErr: false,- },- {- name: "found connection URI, unverified due to error - inactive host",- s: Scanner{},- args: func() args {- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)- defer cancel()- return args{- ctx: ctx,- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, inactiveHost, postgresPort)),- verify: true,- }- }(),- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Postgres,- Verified: false,- Raw: []byte("postgresql://postgres:[email protected]:5434"),- RawV2: []byte("postgresql://postgres:[email protected]:5434"),- ExtraData: map[string]string{"sslmode": "<unset>"},- }- r.SetVerificationError(errors.New("i/o timeout"))- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found connection URI, unverified due to error - wrong port",- s: Scanner{detectLoopback: true},- args: func() args {- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)- defer cancel()- return args{- ctx: ctx,- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s/postgres2`, postgresUser, postgresPass, postgresHost)),- verify: true,- }- }(),- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Postgres,- Verified: false,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),- ExtraData: map[string]string{"sslmode": "<unset>"},- }- r.SetVerificationError(errors.New("connection refused"))- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found connection URI, unverified due to error - ssl not supported (using sslmode)",- s: Scanner{detectLoopback: true},- args: func() args {- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)- defer cancel()- return args{- ctx: ctx,- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=require`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- }- }(),- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Postgres,- Verified: false,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "require"},- }- r.SetVerificationError(pq.ErrSSLNotSupported)- return []detectors.Result{r}- }(),- wantErr: false,- },- {- name: "found connection URI, unverified due to error - ssl not supported (using requiressl)",- s: Scanner{detectLoopback: true},- args: func() args {- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)- defer cancel()- return args{- ctx: ctx,- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=1`, postgresUser, postgresPass, postgresHost, postgresPort)),- verify: true,- }- }(),- want: func() []detectors.Result {- r := detectors.Result{- DetectorType: detectorspb.DetectorType_Postgres,- Verified: false,- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),- ExtraData: map[string]string{"sslmode": "require"},- }- r.SetVerificationError(pq.ErrSSLNotSupported)- return []detectors.Result{r}- }(),- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidUriPattern, keyword, invalidConnStrPartPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- gotErr := ""- if got[i].VerificationError() != nil {- gotErr = got[i].VerificationError().Error()- }- wantErr := ""- if tt.want[i].VerificationError() != nil {- wantErr = tt.want[i].VerificationError().Error()- }- if gotErr != wantErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())- }- }- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "AnalysisInfo")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}-func dockerLogLine(hash string, needle string) chan struct{} {- ch := make(chan struct{}, 1)- go func() {- for {- out, err := exec.Command("docker", "logs", hash).CombinedOutput()+ results, err := d.FromData(context.Background(), false, []byte(test.input))if err != nil {- panic(err)- }- if strings.Contains(string(out), needle) {- ch <- struct{}{}+ t.Errorf("error = %v", err)return}- time.Sleep(1 * time.Second)- }- }()- return ch-}--func startPostgres() error {- cmd := exec.Command(- "docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,- "-e", "POSTGRES_PASSWORD="+postgresPass,- "-e", "POSTGRES_USER="+postgresUser,- "-d", "postgres",- )- fmt.Println(cmd.String())- out, err := cmd.Output()- if err != nil {- return err- }- postgresDockerHash = string(bytes.TrimSpace(out))- select {- case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):- return nil- case <-time.After(30 * time.Second):- stopPostgres()- return errors.New("timeout waiting for postgres database to be ready")- }-}-func stopPostgres() {- exec.Command("docker", "kill", postgresDockerHash).Run()-}+ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/posthog/posthog_integration_test.go b/pkg/detectors/posthog/posthog_integration_test.gonew file mode 100644index 000000000..70091659e--- /dev/null+++ b/pkg/detectors/posthog/posthog_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package posthog++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAppPosthog_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("APPPOSTHOG_TOKEN")+ inactiveSecret := testSecrets.MustGetField("APPPOSTHOG_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appposthog secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PosthogApp,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a appposthog secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PosthogApp,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AppPosthog.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("AppPosthog.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/posthog/posthog_test.go b/pkg/detectors/posthog/posthog_test.goindex 70091659e..923bbc170 100644--- a/pkg/detectors/posthog/posthog_test.go+++ b/pkg/detectors/posthog/posthog_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package posthogimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestAppPosthog_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("APPPOSTHOG_TOKEN")- inactiveSecret := testSecrets.MustGetField("APPPOSTHOG_INACTIVE")+var (+ validPattern = "phx_C1rP9fnAtEFJvb0IYCFdeQhar2WdwUFBYHJym1F_Zqr"+ invalidPattern = "phx_C1rP9fnAtEFJvb0IYCF?eQhar2WdwUFBYHJym1F_Zqr"+ keyword = "posthog"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestAppPosthog_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appposthog secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PosthogApp,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword posthog",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a appposthog secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PosthogApp,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("AppPosthog.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("AppPosthog.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/postman/postman_integration_test.go b/pkg/detectors/postman/postman_integration_test.gonew file mode 100644index 000000000..65ac16c97--- /dev/null+++ b/pkg/detectors/postman/postman_integration_test.go@@ -0,0 +1,158 @@+//go:build detectors+// +build detectors++package postman++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPostman_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POSTMAN_TOKEN")+ inactiveSecret := testSecrets.MustGetField("POSTMAN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postman,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postman,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postman,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postman,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Postman.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Errorf("Postman.FromData() error = %v, wantErr %v", got[i].VerificationError(), tt.wantVerificationErr)+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Postman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/postman/postman_test.go b/pkg/detectors/postman/postman_test.goindex 65ac16c97..10b582888 100644--- a/pkg/detectors/postman/postman_test.go+++ b/pkg/detectors/postman/postman_test.go@@ -1,157 +1,80 @@-//go:build detectors-// +build detectors-package postmanimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPostman_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POSTMAN_TOKEN")- inactiveSecret := testSecrets.MustGetField("POSTMAN_INACTIVE")+var (+ validPattern = "PMAK-aMUVdak2ohpOeeJQVlb1G9EWB09YVQfbvH6YH8HpPRkXHPBg5qYvp9YGOhI"+ invalidPattern = "PMAK-aMUVdak2ohpOeeJQVlb1G9EWB09?VQfbvH6YH8HpPRkXHPBg5qYvp9YGOhI"+ keyword = "postman"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPostman_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postman,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postman,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postman,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postman,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - with keyword postman",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Postman.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Errorf("Postman.FromData() error = %v, wantErr %v", got[i].VerificationError(), tt.wantVerificationErr)+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Postman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/postmark/postmark_integration_test.go b/pkg/detectors/postmark/postmark_integration_test.gonew file mode 100644index 000000000..a9c984c7c--- /dev/null+++ b/pkg/detectors/postmark/postmark_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package postmark++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPostmark_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POSTMARK_TOKEN")+ inactiveSecret := testSecrets.MustGetField("POSTMARK_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postmark secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postmark,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a postmark secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Postmark,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Postmark.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Postmark.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/postmark/postmark_test.go b/pkg/detectors/postmark/postmark_test.goindex a9c984c7c..8c0e4b6b7 100644--- a/pkg/detectors/postmark/postmark_test.go+++ b/pkg/detectors/postmark/postmark_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package postmarkimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPostmark_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POSTMARK_TOKEN")- inactiveSecret := testSecrets.MustGetField("POSTMARK_INACTIVE")+var (+ validPattern = "uigi9zkt-7q3u-v1v9-5x2s-91p78wlmsuxv"+ invalidPattern = "uigi9zkt?7q3u-v1v9-5x2s-91p78wlmsuxv"+ keyword = "postmark"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPostmark_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postmark secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postmark,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword postmark",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},+ },+ {+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a postmark secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Postmark,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Postmark.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Postmark.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/powrbot/powrbot_integration_test.go b/pkg/detectors/powrbot/powrbot_integration_test.gonew file mode 100644index 000000000..0b7b184ab--- /dev/null+++ b/pkg/detectors/powrbot/powrbot_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package powrbot++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPowrbot_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("POWRBOT_TOKEN")+ inactiveSecret := testSecrets.MustGetField("POWRBOT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a powrbot secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Powrbot,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a powrbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Powrbot,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Powrbot.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Powrbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/powrbot/powrbot_test.go b/pkg/detectors/powrbot/powrbot_test.goindex 0b7b184ab..117a1efd0 100644--- a/pkg/detectors/powrbot/powrbot_test.go+++ b/pkg/detectors/powrbot/powrbot_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package powrbotimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPowrbot_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("POWRBOT_TOKEN")- inactiveSecret := testSecrets.MustGetField("POWRBOT_INACTIVE")+var (+ validPattern = "BuvR1k59n67R96aInATbIUbchmLcpLVN10oyqKuB"+ invalidPattern = "BuvR1k59n67R96a?nATbIUbchmLcpLVN10oyqKuB"+ keyword = "powrbot"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPowrbot_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a powrbot secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Powrbot,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword powrbot",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a powrbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Powrbot,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Powrbot.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Powrbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/prefect/prefect_integration_test.go b/pkg/detectors/prefect/prefect_integration_test.gonew file mode 100644index 000000000..d748c86de--- /dev/null+++ b/pkg/detectors/prefect/prefect_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package prefect++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPrefect_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PREFECT")+ inactiveSecret := testSecrets.MustGetField("PREFECT_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a prefect secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Prefect,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a prefect secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Prefect,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Prefect.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Prefect.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/prefect/prefect_test.go b/pkg/detectors/prefect/prefect_test.goindex d748c86de..10aa2e077 100644--- a/pkg/detectors/prefect/prefect_test.go+++ b/pkg/detectors/prefect/prefect_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package prefectimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPrefect_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PREFECT")- inactiveSecret := testSecrets.MustGetField("PREFECT_INACTIVE")+var (+ validPattern = "pnu_v7mBGSR1R1WT0He5KS2w83RZgInZO8Qwalvr"+ invalidPattern = "pnu_v7mBGSR1R1WT0He5?S2w83RZgInZO8Qwalvr"+ keyword = "prefect"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPrefect_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a prefect secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Prefect,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword prefect",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a prefect secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Prefect,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Prefect.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Prefect.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/privacy/privacy_integration_test.go b/pkg/detectors/privacy/privacy_integration_test.gonew file mode 100644index 000000000..c17317ea6--- /dev/null+++ b/pkg/detectors/privacy/privacy_integration_test.go@@ -0,0 +1,162 @@+//go:build detectors+// +build detectors++package privacy++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPrivacy_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PRIVACY")+ inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Privacy,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Privacy,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Privacy,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Privacy,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/privacy/privacy_test.go b/pkg/detectors/privacy/privacy_test.goindex c17317ea6..540db09ca 100644--- a/pkg/detectors/privacy/privacy_test.go+++ b/pkg/detectors/privacy/privacy_test.go@@ -1,161 +1,90 @@-//go:build detectors-// +build detectors-package privacyimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPrivacy_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PRIVACY")- inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")+var (+ validPattern = "sccqe60k-nsak-h1m1-vzq8-vr7u4gxihiwl"+ invalidPattern = "sccqe60k?nsak-h1m1-vzq8-vr7u4gxihiwl"+ keyword = "privacy"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPrivacy_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Privacy,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - with keyword privacy",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Privacy,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},},{- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Privacy,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Privacy,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/privatekey/privatekey_integration_test.go b/pkg/detectors/privatekey/privatekey_integration_test.gonew file mode 100644index 000000000..44c9cf4f9--- /dev/null+++ b/pkg/detectors/privatekey/privatekey_integration_test.go@@ -0,0 +1,216 @@+//go:build detectors+// +build detectors++package privatekey++import (+ "context"+ "fmt"+ "os"+ "reflect"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPrivatekey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")+ secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")+ secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")+ secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }++ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: false,+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",+ },+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: false,+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",+ },+ },+ wantErr: false,+ },+ {+ name: "found TLS private key, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(secretTLS),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: true,+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",+ ExtraData: map[string]string{+ "certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found GitHub SSH private key, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(secretGitHub),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: true,+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",+ ExtraData: map[string]string{+ "github_user": "sirdetectsalot",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found encrypted GitHub SSH private key, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(secretGitHubEncrypted),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PrivateKey,+ Verified: true,+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",+ ExtraData: map[string]string{+ "github_user": "sirdetectsalot",+ "encrypted": "true",+ "cracked_encryption_passphrase": "true",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{IncludeExpired: true}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if os.Getenv("FORCE_PASS_DIFF") == "true" {+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}++func Test_lookupFingerprint(t *testing.T) {+ tests := []struct {+ name string+ publicKeyFingerprintInHex string+ wantFingerprints bool+ wantErr bool+ includeExpired bool+ }{+ {+ name: "got some",+ publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",+ wantFingerprints: true,+ wantErr: false,+ includeExpired: true,+ },+ {+ name: "got some",+ publicKeyFingerprintInHex: "none",+ wantFingerprints: false,+ wantErr: false,+ includeExpired: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ gotFingerprints, err := lookupFingerprint(context.TODO(), tt.publicKeyFingerprintInHex, tt.includeExpired)+ if (err != nil) != tt.wantErr {+ t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {+ t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)+ }+ })+ }+}diff --git a/pkg/detectors/privatekey/privatekey_test.go b/pkg/detectors/privatekey/privatekey_test.goindex 44c9cf4f9..661a8fd96 100644--- a/pkg/detectors/privatekey/privatekey_test.go+++ b/pkg/detectors/privatekey/privatekey_test.go@@ -1,215 +1,97 @@-//go:build detectors-// +build detectors-package privatekeyimport ("context""fmt"- "os"- "reflect""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPrivatekey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")- secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")- secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")- secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")-- type args struct {- ctx context.Context- data []byte- verify bool- }+var (+ validPattern = `-----BEGIN RSA PRIVATE KEY-----+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00+-----END RSA PRIVATE KEY-----+`+ invalidPattern = `-----BEGIN?RSA?PRIVATE?KEY-----+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00+-----END RSA PRIVATE KEY-----`+ keyword = "privatekey"+)+func TestPrivatekey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PrivateKey,- Verified: false,- Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",- },- {- DetectorType: detectorspb.DetectorType_PrivateKey,- Verified: false,- Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",- },- },- wantErr: false,- },- {- name: "found TLS private key, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(secretTLS),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PrivateKey,- Verified: true,- Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",- ExtraData: map[string]string{- "certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",- },- },- },- wantErr: false,+ name: "valid pattern - with keyword privatekey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found GitHub SSH private key, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(secretGitHub),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PrivateKey,- Verified: true,- Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",- ExtraData: map[string]string{- "github_user": "sirdetectsalot",- },- },- },- wantErr: false,- },- {- name: "found encrypted GitHub SSH private key, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(secretGitHubEncrypted),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PrivateKey,- Verified: true,- Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",- ExtraData: map[string]string{- "github_user": "sirdetectsalot",- "encrypted": "true",- "cracked_encryption_passphrase": "true",- },- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{IncludeExpired: true}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- if os.Getenv("FORCE_PASS_DIFF") == "true" {++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- })- }-}--func Test_lookupFingerprint(t *testing.T) {- tests := []struct {- name string- publicKeyFingerprintInHex string- wantFingerprints bool- wantErr bool- includeExpired bool- }{- {- name: "got some",- publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",- wantFingerprints: true,- wantErr: false,- includeExpired: true,- },- {- name: "got some",- publicKeyFingerprintInHex: "none",- wantFingerprints: false,- wantErr: false,- includeExpired: true,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- gotFingerprints, err := lookupFingerprint(context.TODO(), tt.publicKeyFingerprintInHex, tt.includeExpired)- if (err != nil) != tt.wantErr {- t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)- return+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {- t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/prodpad/prodpad_integration_test.go b/pkg/detectors/prodpad/prodpad_integration_test.gonew file mode 100644index 000000000..80fd2a5e7--- /dev/null+++ b/pkg/detectors/prodpad/prodpad_integration_test.go@@ -0,0 +1,117 @@+package prodpad++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestProdpad_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PRODPAD")+ inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Prodpad,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Prodpad,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/prodpad/prodpad_test.go b/pkg/detectors/prodpad/prodpad_test.goindex 80fd2a5e7..2e5deb815 100644--- a/pkg/detectors/prodpad/prodpad_test.go+++ b/pkg/detectors/prodpad/prodpad_test.go@@ -4,114 +4,88 @@ import ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestProdpad_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PRODPAD")- inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")+var (+ validPattern = "96d3a2234e30b05b6c0ec74df1943b95ec708d09b0c45779874be404ce004335"+ invalidPattern = "96d3a2234?30b05b6c0ec74df1943b95ec708d09b0c45779874be404ce004335"+ keyword = "prodpad"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestProdpad_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Prodpad,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword prodpad",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Prodpad,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/prospectcrm/prospectcrm_integration_test.go b/pkg/detectors/prospectcrm/prospectcrm_integration_test.gonew file mode 100644index 000000000..af0f8f0fb--- /dev/null+++ b/pkg/detectors/prospectcrm/prospectcrm_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package prospectcrm++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestProspectCRM_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PROSPECTCRM")+ inactiveSecret := testSecrets.MustGetField("PROSPECTCRM_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProspectCRM,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProspectCRM,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ProspectCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ProspectCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/prospectcrm/prospectcrm_test.go b/pkg/detectors/prospectcrm/prospectcrm_test.goindex af0f8f0fb..86175b5af 100644--- a/pkg/detectors/prospectcrm/prospectcrm_test.go+++ b/pkg/detectors/prospectcrm/prospectcrm_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package prospectcrmimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestProspectCRM_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PROSPECTCRM")- inactiveSecret := testSecrets.MustGetField("PROSPECTCRM_INACTIVE")+var (+ validPattern = "yawgnnmzs6hqcpdakzpshc79enh55iz0"+ invalidPattern = "yawgnnmzs6hqcpda?zpshc79enh55iz0"+ keyword = "prospectcrm"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestProspectCRM_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ProspectCRM,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword prospectcrm",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ProspectCRM,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ProspectCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ProspectCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/protocolsio/protocolsio_integration_test.go b/pkg/detectors/protocolsio/protocolsio_integration_test.gonew file mode 100644index 000000000..495d2218f--- /dev/null+++ b/pkg/detectors/protocolsio/protocolsio_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package protocolsio++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestProtocolsIO_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PROTOCOLSIO")+ inactiveSecret := testSecrets.MustGetField("PROTOCOLSIO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProtocolsIO,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProtocolsIO,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ProtocolsIO.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ProtocolsIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/protocolsio/protocolsio_test.go b/pkg/detectors/protocolsio/protocolsio_test.goindex 495d2218f..66d7ae639 100644--- a/pkg/detectors/protocolsio/protocolsio_test.go+++ b/pkg/detectors/protocolsio/protocolsio_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package protocolsioimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestProtocolsIO_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PROTOCOLSIO")- inactiveSecret := testSecrets.MustGetField("PROTOCOLSIO_INACTIVE")+var (+ validPattern = "4aphfkag41i9kow1la6gm7sj9zsj5i8xznwlg5j9eq0e1xl3dzr44b19llppaiut"+ invalidPattern = "4aphfkag41i9kow1la6gm7sj9?sj5i8xznwlg5j9eq0e1xl3dzr44b19llppaiut"+ keyword = "protocolsio"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestProtocolsIO_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ProtocolsIO,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword protocolsio",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ProtocolsIO,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ProtocolsIO.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ProtocolsIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/proxycrawl/proxycrawl_integration_test.go b/pkg/detectors/proxycrawl/proxycrawl_integration_test.gonew file mode 100644index 000000000..7eea9e70a--- /dev/null+++ b/pkg/detectors/proxycrawl/proxycrawl_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package proxycrawl++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestProxyCrawl_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PROXYCRAWL")+ inactiveSecret := testSecrets.MustGetField("PROXYCRAWL_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProxyCrawl,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_ProxyCrawl,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("ProxyCrawl.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("ProxyCrawl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/proxycrawl/proxycrawl_test.go b/pkg/detectors/proxycrawl/proxycrawl_test.goindex 7eea9e70a..31faaa415 100644--- a/pkg/detectors/proxycrawl/proxycrawl_test.go+++ b/pkg/detectors/proxycrawl/proxycrawl_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package proxycrawlimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestProxyCrawl_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PROXYCRAWL")- inactiveSecret := testSecrets.MustGetField("PROXYCRAWL_INACTIVE")+var (+ validPattern = "wjXKCFzOWEPRqhTmmbWtVQ"+ invalidPattern = "wjXKCFzOWEP?qhTmmbWtVQ"+ keyword = "proxycrawl"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestProxyCrawl_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ProxyCrawl,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword proxycrawl",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ProxyCrawl,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ProxyCrawl.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ProxyCrawl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_integration_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_integration_test.gonew file mode 100644index 000000000..9caa04ae1--- /dev/null+++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_integration_test.go@@ -0,0 +1,160 @@+//go:build detectors+// +build detectors++package pubnubpublishkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPubNubPublishKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ pub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_TOKEN")+ inactivePub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_INACTIVE")+ sub := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s but not valid", inactivePub, sub)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "found, would be verified but for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, valid but unexpected api response",+ s: Scanner{client: common.ConstantResponseHttpClient(500, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ if diff := cmp.Diff(got, tt.want, opts); diff != "" {+ t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.goindex 9caa04ae1..279b26df6 100644--- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go+++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go@@ -1,159 +1,82 @@-//go:build detectors-// +build detectors-package pubnubpublishkeyimport ("context""fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPubNubPublishKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- pub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_TOKEN")- inactivePub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_INACTIVE")- sub := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")+var (+ validPub = "pub-c-3886bd4t-dwor-vuqm-ond2-4wz4supiwwa2"+ invalidPub = "p?b-c-3886bd4t-dwor-vuqm-ond2-4wz4supiwwa2"+ validSub = "sub-c-adky624g-bux6-lz1j-8yrx-8fvodaul9hu5"+ invalidSub = "sub-c-adky624g-bux6-l?1j-8yrx-8fvodaul9hu5"+ keyword = "pubnubpublishkey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPubNubPublishKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PubNubPublishKey,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s but not valid", inactivePub, sub)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PubNubPublishKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "found, would be verified but for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PubNubPublishKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,+ name: "valid pattern - with keyword pubnubpublishkey",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validPub, keyword, validSub),+ want: []string{validPub + "/" + validSub},},{- name: "found, valid but unexpected api response",- s: Scanner{client: common.ConstantResponseHttpClient(500, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PubNubPublishKey,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidPub, keyword, invalidSub),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))}- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")- if diff := cmp.Diff(got, tt.want, opts); diff != "" {- t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }+ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)}})}diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_integration_test.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_integration_test.gonew file mode 100644index 000000000..bbeac7afb--- /dev/null+++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pubnubsubscriptionkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPubNubSubscriptionKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PubNubSubscriptionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PubNubSubscriptionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.goindex bbeac7afb..54b6b93ac 100644--- a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go+++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package pubnubsubscriptionkeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPubNubSubscriptionKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_INACTIVE")+var (+ validPattern = "sub-c-sy4te40h-w1oc-zpyq-zlrw-w6ehv0y0y78c"+ invalidPattern = "sub-c-sy4te40h-w1oc-z?yq-zlrw-w6ehv0y0y78c"+ keyword = "pubnubsubscriptionkey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPubNubSubscriptionKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pubnubsubscriptionkey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PubNubSubscriptionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PubNubSubscriptionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pulumi/pulumi_integration_test.go b/pkg/detectors/pulumi/pulumi_integration_test.gonew file mode 100644index 000000000..2276bddcf--- /dev/null+++ b/pkg/detectors/pulumi/pulumi_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pulumi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPulumi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PULUMI")+ inactiveSecret := testSecrets.MustGetField("PULUMI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pulumi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pulumi,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pulumi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Pulumi,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pulumi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Pulumi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pulumi/pulumi_test.go b/pkg/detectors/pulumi/pulumi_test.goindex 2276bddcf..9783781f1 100644--- a/pkg/detectors/pulumi/pulumi_test.go+++ b/pkg/detectors/pulumi/pulumi_test.go@@ -1,120 +1,81 @@-//go:build detectors-// +build detectors-package pulumiimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPulumi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PULUMI")- inactiveSecret := testSecrets.MustGetField("PULUMI_INACTIVE")+var (+ validPattern = "pul-xm8r0t347qgoqhrp4c5pq0jkkabeue0x7u35l2ac"+ invalidPattern = "pul-xm8r0t347qgoqhrp4c?pq0jkkabeue0x7u35l2ac"+ keyword = "pulumi"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPulumi_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pulumi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pulumi,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pulumi",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pulumi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Pulumi,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pulumi.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Pulumi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/purestake/purestake_integration_test.go b/pkg/detectors/purestake/purestake_integration_test.gonew file mode 100644index 000000000..90c408e42--- /dev/null+++ b/pkg/detectors/purestake/purestake_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package purestake++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPureStake_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PURESTAKE")+ inactiveSecret := testSecrets.MustGetField("PURESTAKE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a purestake secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PureStake,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a purestake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PureStake,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PureStake.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PureStake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/purestake/purestake_test.go b/pkg/detectors/purestake/purestake_test.goindex 90c408e42..8d2e72cb7 100644--- a/pkg/detectors/purestake/purestake_test.go+++ b/pkg/detectors/purestake/purestake_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package purestakeimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPureStake_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PURESTAKE")- inactiveSecret := testSecrets.MustGetField("PURESTAKE_INACTIVE")+var (+ validPattern = "uR05tbMNjbrrHCVDGinuNn54QqYiSiSZn5BN3AGW"+ invalidPattern = "uR05tbM?jbrrHCVDGinuNn54QqYiSiSZn5BN3AGW"+ keyword = "purestake"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPureStake_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a purestake secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PureStake,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword purestake",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a purestake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PureStake,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PureStake.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PureStake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey_integration_test.go b/pkg/detectors/pushbulletapikey/pushbulletapikey_integration_test.gonew file mode 100644index 000000000..81aeb603d--- /dev/null+++ b/pkg/detectors/pushbulletapikey/pushbulletapikey_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package pushbulletapikey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPushBulletApiKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PUSHBULLETAPIKEY_TOKEN")+ inactiveSecret := testSecrets.MustGetField("PUSHBULLETAPIKEY_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PushBulletApiKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PushBulletApiKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PushBulletApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PushBulletApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.goindex 81aeb603d..645ccd65f 100644--- a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go+++ b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package pushbulletapikeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPushBulletApiKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PUSHBULLETAPIKEY_TOKEN")- inactiveSecret := testSecrets.MustGetField("PUSHBULLETAPIKEY_INACTIVE")+var (+ validPattern = "S2QPKsMCE3hpR1ac2HYl5EvzdvbrugJCpP"+ invalidPattern = "S2QPKsMCE3hpR1ac2?Yl5EvzdvbrugJCpP"+ keyword = "pushbulletapikey"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPushBulletApiKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PushBulletApiKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword pushbulletapikey",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PushBulletApiKey,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PushBulletApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PushBulletApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey_integration_test.go b/pkg/detectors/pusherchannelkey/pusherchannelkey_integration_test.gonew file mode 100644index 000000000..e6eac1483--- /dev/null+++ b/pkg/detectors/pusherchannelkey/pusherchannelkey_integration_test.go@@ -0,0 +1,122 @@+//go:build detectors+// +build detectors++package pusherchannelkey++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPusherChannelKey_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ appId := testSecrets.MustGetField("PUSHER_APPID")+ key := testSecrets.MustGetField("PUSHER_KEY")+ secret := testSecrets.MustGetField("PUSHER_SECRET")+ inactiveSecret := testSecrets.MustGetField("PUSHER_SECRET_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s within pusherKey %s", secret, appId, key)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PusherChannelKey,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s but not pusherKey %s valid", inactiveSecret, appId, key)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PusherChannelKey,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("PusherChannelKey.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("PusherChannelKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.goindex e6eac1483..cd0e9e01f 100644--- a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go+++ b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go@@ -1,122 +1,87 @@-//go:build detectors-// +build detectors-package pusherchannelkeyimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestPusherChannelKey_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- appId := testSecrets.MustGetField("PUSHER_APPID")- key := testSecrets.MustGetField("PUSHER_KEY")- secret := testSecrets.MustGetField("PUSHER_SECRET")- inactiveSecret := testSecrets.MustGetField("PUSHER_SECRET_INACTIVE")+var (+ validAppId = "4870172"+ invalidAppId = "487?172"+ validKey = "8mcc284l3i4sph5a3mgf"+ invalidKey = "8mcc28?l3i4sph5a3mgf"+ validSecret = "pg1podgq3qabvh4yd1np"+ invalidSecret = "pg1podgq3q?bvh4yd1np"+ validInput = `pusher - '%s'+ key - '%s'+ pusher - '%s'`+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestPusherChannelKey_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s within pusherKey %s", secret, appId, key)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PusherChannelKey,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern",+ input: fmt.Sprintf(validInput, validAppId, validKey, validSecret),+ want: []string{validAppId + validKey},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s but not pusherKey %s valid", inactiveSecret, appId, key)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PusherChannelKey,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: fmt.Sprintf(validInput, invalidAppId, invalidKey, invalidSecret),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("PusherChannelKey.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("PusherChannelKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/pypi/pypi_integration_test.go b/pkg/detectors/pypi/pypi_integration_test.gonew file mode 100644index 000000000..dddf57e41--- /dev/null+++ b/pkg/detectors/pypi/pypi_integration_test.go@@ -0,0 +1,161 @@+//go:build detectors+// +build detectors++package pypi++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestPypi_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("PYPI")+ inactiveSecret := testSecrets.MustGetField("PYPI_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PyPI,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PyPI,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, would be verified if not for timeout",+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PyPI,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ {+ name: "found, verified but unexpected api surface",+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_PyPI,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Pypi.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("Pypi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/pypi/pypi_test.go b/pkg/detectors/pypi/pypi_test.goindex 614da0756..d5c755c0f 100644--- a/pkg/detectors/pypi/pypi_test.go+++ b/pkg/detectors/pypi/pypi_test.go@@ -1,21 +1,13 @@-//go:build detectors-// +build detectors-package pypiimport ("context"- "fmt""testing"- "time""github.com/google/go-cmp/cmp"- "github.com/google/go-cmp/cmp/cmpopts"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")func TestPypi_Pattern(t *testing.T) {@@ -75,146 +67,3 @@ func TestPypi_Pattern(t *testing.T) {})}}--func TestPypi_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("PYPI")- inactiveSecret := testSecrets.MustGetField("PYPI_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- wantVerificationErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PyPI,- Verified: true,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pypi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PyPI,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- wantVerificationErr: false,- },- {- name: "found, would be verified if not for timeout",- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PyPI,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- {- name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_PyPI,- Verified: false,- },- },- wantErr: false,- wantVerificationErr: true,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Pypi.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())- }- }- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {- t.Errorf("Pypi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }- }- })- }-}diff --git a/pkg/detectors/qase/qase_integration_test.go b/pkg/detectors/qase/qase_integration_test.gonew file mode 100644index 000000000..271d7d2f9--- /dev/null+++ b/pkg/detectors/qase/qase_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package qase++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestQase_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("QASE")+ inactiveSecret := testSecrets.MustGetField("QASE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a qase secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Qase,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a qase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Qase,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Qase.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Qase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/qase/qase_test.go b/pkg/detectors/qase/qase_test.goindex 271d7d2f9..e2f89299c 100644--- a/pkg/detectors/qase/qase_test.go+++ b/pkg/detectors/qase/qase_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package qaseimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestQase_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("QASE")- inactiveSecret := testSecrets.MustGetField("QASE_INACTIVE")+var (+ validPattern = "X8FDrZHoBELPU4UpeeSIMyQPEi7JyQFsF6oQZi6X"+ invalidPattern = "X8FDrZHoBELPU4UpeeSI?yQPEi7JyQFsF6oQZi6X"+ keyword = "qase"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestQase_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a qase secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Qase,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword qase",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a qase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Qase,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Qase.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Qase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/qualaroo/qualaroo_integration_test.go b/pkg/detectors/qualaroo/qualaroo_integration_test.gonew file mode 100644index 000000000..92637db43--- /dev/null+++ b/pkg/detectors/qualaroo/qualaroo_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package qualaroo++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestQualaroo_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("QUALAROO_TOKEN")+ inactiveSecret := testSecrets.MustGetField("QUALAROO_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Qualaroo,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Qualaroo,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Qualaroo.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Qualaroo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/qualaroo/qualaroo_test.go b/pkg/detectors/qualaroo/qualaroo_test.goindex 92637db43..0942f1b14 100644--- a/pkg/detectors/qualaroo/qualaroo_test.go+++ b/pkg/detectors/qualaroo/qualaroo_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package qualarooimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestQualaroo_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("QUALAROO_TOKEN")- inactiveSecret := testSecrets.MustGetField("QUALAROO_INACTIVE")+var (+ validPattern = "w82SrLORVOsr4VMrPhDG4fih0rYGw86vxGBZch99TVvTfiSBSlpWF6f42BVtf0jI"+ invalidPattern = "w82SrLORVOsr4VMrPhDG4fih0rYGw86v?GBZch99TVvTfiSBSlpWF6f42BVtf0jI"+ keyword = "qualaroo"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestQualaroo_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Qualaroo,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword qualaroo",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Qualaroo,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Qualaroo.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Qualaroo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}diff --git a/pkg/detectors/qubole/qubole_integration_test.go b/pkg/detectors/qubole/qubole_integration_test.gonew file mode 100644index 000000000..ccd8c83e0--- /dev/null+++ b/pkg/detectors/qubole/qubole_integration_test.go@@ -0,0 +1,120 @@+//go:build detectors+// +build detectors++package qubole++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestQubole_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("QUBOLE")+ inactiveSecret := testSecrets.MustGetField("QUBOLE_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a qubole secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Qubole,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a qubole secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Qubole,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("Qubole.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("Qubole.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/qubole/qubole_test.go b/pkg/detectors/qubole/qubole_test.goindex ccd8c83e0..804b49c34 100644--- a/pkg/detectors/qubole/qubole_test.go+++ b/pkg/detectors/qubole/qubole_test.go@@ -1,120 +1,91 @@-//go:build detectors-// +build detectors-package quboleimport ("context""fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/google/go-cmp/cmp"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-func TestQubole_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("QUBOLE")- inactiveSecret := testSecrets.MustGetField("QUBOLE_INACTIVE")+var (+ validPattern = "8p9tm1c0ejc70n8vpef7nq7agfzq7kks73lwxpo29z25ggxszg22ecl9toewbdob"+ invalidPattern = "8p9tm1c0ejc?0n8vpef7nq7agfzq7kks73lwxpo29z25ggxszg22ecl9toewbdob"+ keyword = "qubole"+)- type args struct {- ctx context.Context- data []byte- verify bool- }+func TestQubole_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a qubole secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Qubole,- Verified: true,- },- },- wantErr: false,+ name: "valid pattern - with keyword qubole",+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),+ want: []string{validPattern},},{- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a qubole secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Qubole,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern - ignore duplicate",+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),+ want: []string{validPattern},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "valid pattern - key out of prefix range",+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),+ want: []string{},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),+ want: []string{},},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Qubole.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Qubole.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}