Skip to content

Find, verify, and analyze leaked credentials

added and updated pattern tests for detectors starting from p to q (#3710)

Nabeel Alam35943b41905e
diff --git a/pkg/detectors/packagecloud/packagecloud_integration_test.go b/pkg/detectors/packagecloud/packagecloud_integration_test.go
new file mode 100644
index 000000000..ca8dc5c79
--- /dev/null
+++ b/pkg/detectors/packagecloud/packagecloud_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package packagecloud
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPackageCloud_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PACKAGECLOUD")
+ inactiveSecret := testSecrets.MustGetField("PACKAGECLOUD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PackageCloud,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PackageCloud,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PackageCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PackageCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/packagecloud/packagecloud_test.go b/pkg/detectors/packagecloud/packagecloud_test.go
index ca8dc5c79..a088ba827 100644
--- a/pkg/detectors/packagecloud/packagecloud_test.go
+++ b/pkg/detectors/packagecloud/packagecloud_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package packagecloud
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPackageCloud_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PACKAGECLOUD")
- inactiveSecret := testSecrets.MustGetField("PACKAGECLOUD_INACTIVE")
+var (
+ validPattern = "3a4c163d0c145e7346b53f6d9be8e0a18058a5b7f03e2b41"
+ invalidPattern = "3a4c163d0c145e7346b53f6d?be8e0a18058a5b7f03e2b41"
+ keyword = "packagecloud"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPackageCloud_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PackageCloud,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword packagecloud",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a packagecloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PackageCloud,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PackageCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PackageCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go
new file mode 100644
index 000000000..f3a1ff55c
--- /dev/null
+++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_integration_test.go
@@ -0,0 +1,157 @@
+//go:build detectors
+// +build detectors
+
+package pagerdutyapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPagerDutyApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAGERDUTYAPIKEY_TOKEN")
+ invalidSecret := testSecrets.MustGetField("PAGERDUTYAPIKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within but not valid", invalidSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PagerDutyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Errorf("PagerDutyApiKey.FromData() verificationError = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr)
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("PagerDutyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go
index f3a1ff55c..bc00c25b0 100644
--- a/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go
+++ b/pkg/detectors/pagerdutyapikey/pagerdutyapikey_test.go
@@ -1,156 +1,90 @@
-//go:build detectors
-// +build detectors
-
package pagerdutyapikey
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPagerDutyApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAGERDUTYAPIKEY_TOKEN")
- invalidSecret := testSecrets.MustGetField("PAGERDUTYAPIKEY_INACTIVE")
+var (
+ validPattern = "u+C4lGv0tct7TlvSUDoc"
+ invalidPattern = "u+C4lGv0?ct7TlvSUDoc"
+ keyword = "pagerdutyapikey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPagerDutyApiKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pagerdutyapikey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pagerdutyapikey secret %s within but not valid", invalidSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PagerDutyApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PagerDutyApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Errorf("PagerDutyApiKey.FromData() verificationError = %v, wantVerificationErr %v", got[i].VerificationError(), tt.wantVerificationErr)
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("PagerDutyApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/pandadoc/pandadoc_integration_test.go b/pkg/detectors/pandadoc/pandadoc_integration_test.go
new file mode 100644
index 000000000..52f140a88
--- /dev/null
+++ b/pkg/detectors/pandadoc/pandadoc_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pandadoc
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPandadoc_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PANDADOC")
+ inactiveSecret := testSecrets.MustGetField("PANDADOC_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pandadoc,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pandadoc,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pandadoc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pandadoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pandadoc/pandadoc_test.go b/pkg/detectors/pandadoc/pandadoc_test.go
index 52f140a88..d68ad4c8c 100644
--- a/pkg/detectors/pandadoc/pandadoc_test.go
+++ b/pkg/detectors/pandadoc/pandadoc_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pandadoc
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPandadoc_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PANDADOC")
- inactiveSecret := testSecrets.MustGetField("PANDADOC_INACTIVE")
+var (
+ validPattern = "gL1LKFp8p0q7ORUybiyovqIuGUsGjx4adxMRcHPh"
+ invalidPattern = "gL1LKFp8p0q7ORUybiyo?qIuGUsGjx4adxMRcHPh"
+ keyword = "pandadoc"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPandadoc_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pandadoc,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pandadoc",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pandadoc secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pandadoc,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pandadoc.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pandadoc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pandascore/pandascore_integration_test.go b/pkg/detectors/pandascore/pandascore_integration_test.go
new file mode 100644
index 000000000..b6d5d005b
--- /dev/null
+++ b/pkg/detectors/pandascore/pandascore_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package pandascore
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPandaScore_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PANDASCORE")
+ inactiveSecret := testSecrets.MustGetField("PANDASCORE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pandascore secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PandaScore,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pandascore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PandaScore,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PandaScore.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PandaScore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pandascore/pandascore_test.go b/pkg/detectors/pandascore/pandascore_test.go
index b6d5d005b..46922fd00 100644
--- a/pkg/detectors/pandascore/pandascore_test.go
+++ b/pkg/detectors/pandascore/pandascore_test.go
@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
package pandascore
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPandaScore_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PANDASCORE")
- inactiveSecret := testSecrets.MustGetField("PANDASCORE_INACTIVE")
+var (
+ validPattern = "iahNbqgnxq7tzca9hNBWiVexHeePoQ2nnXwN06wmaFSY5BMGyfq"
+ invalidPattern = "iahNbqgnxq7tzca9hNBWiV?xHeePoQ2nnXwN06wmaFSY5BMGyfq"
+ keyword = "pandascore"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPandaScore_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pandascore secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PandaScore,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pandascore",
+ input: fmt.Sprintf("%s token = ' %s '", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pandascore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PandaScore,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = ' %s ' | ' %s '", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = ' %s '", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = ' %s '", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PandaScore.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PandaScore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/paperform/paperform_integration_test.go b/pkg/detectors/paperform/paperform_integration_test.go
new file mode 100644
index 000000000..2b3de57e2
--- /dev/null
+++ b/pkg/detectors/paperform/paperform_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paperform
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaperform_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAPERFORM")
+ inactiveSecret := testSecrets.MustGetField("PAPERFORM_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paperform secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paperform,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paperform secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paperform,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paperform.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paperform.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paperform/paperform_test.go b/pkg/detectors/paperform/paperform_test.go
index 2b3de57e2..23fc08e03 100644
--- a/pkg/detectors/paperform/paperform_test.go
+++ b/pkg/detectors/paperform/paperform_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package paperform
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPaperform_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAPERFORM")
- inactiveSecret := testSecrets.MustGetField("PAPERFORM_INACTIVE")
+var (
+ validPattern = "NvBbE5vYVPhidc-tiZ7E6DvP.z4bAQj29KEZOCv_XEl1pIn.Y1q8JoeGNoSelGj.db1iQpLM0fsype86LD.Vk.p6yweF5A2MlfxXDrEd7nz2SBExnTpp4QRN94pBAeulPLLtzqTz--y_UuF1g6cjE_.kuW_u5He0QkBdLMkyAgSx94N3Csj9LY37XOmUp9IIi9LXpTvZGa8oywp6JuMfhzwg5OCEvdp9mx.UyfQcnnJtYzzP5dItmwfEC-KJIvjvS8LF2NU2w2japQHtnAJqBAn3_EP-FN78wHnDEWANANT0cfor6kDqyKraO0Y-26PdB6xBjm3_VpU.8hnKIyoKdLQ6S.HZwr5rx0Bx76zXTCBv4uEzhtDFcDqVPN8ZG_kE90P..ldReG0jU4w3YA2jbaOgi6i-8llYWGoCiFFBm3Od-zLOEDYL2BlGsUFRUkiEMjytCVDqcIOdfPT7GQcd3wdmort6FFv8SbCu95f2gBCcM.5.ZmMxIOybubMGmiRunYM8-pSaVvXfBQSkM2Eygh15tkKCDHf8X3InAkPh7HQn13mP5y1gFRLsVAUWb-91PeHASP6hluUEdsX3uLQ9OJFenKrk.0zS9Goy08bfttd4h4Jtb2JV8vbJ8-3Wb4AJWqf0eUALMxOChB3sSBKW37s4vDb1NKOnoqOeoYQUBijqRGu9YLKIAimwo7Uvl0CuD7bWNrERweBqNVWjfGhlE8Yvvklm5YhCk5XY02pOa3IjMf_TDKhbTr8bh_20SXevnDk80XKg_3mWbhuieL23kx835AokAg9JpEkgydBBqo8nzQg23R5xJzKRT64kgb5GBlzuM9Oxh7pXsmzlYf"
+ invalidPattern = "NvBbE5vYVPhidc?tiZ7E6DvP.z4bAQj29KEZOCv_XEl1pIn.Y1q8JoeGNoSelGj.db1iQpLM0fsype86LD.Vk.p6yweF5A2MlfxXDrEd7nz2SBExnTpp4QRN94pBAeulPLLtzqTz--y_UuF1g6cjE_.kuW_u5He0QkBdLMkyAgSx94N3Csj9LY37XOmUp9IIi9LXpTvZGa8oywp6JuMfhzwg5OCEvdp9mx.UyfQcnnJtYzzP5dItmwfEC-KJIvjvS8LF2NU2w2japQHtnAJqBAn3_EP-FN78wHnDEWANANT0cfor6kDqyKraO0Y-26PdB6xBjm3_VpU.8hnKIyoKdLQ6S.HZwr5rx0Bx76zXTCBv4uEzhtDFcDqVPN8ZG_kE90P..ldReG0jU4w3YA2jbaOgi6i-8llYWGoCiFFBm3Od-zLOEDYL2BlGsUFRUkiEMjytCVDqcIOdfPT7GQcd3wdmort6FFv8SbCu95f2gBCcM.5.ZmMxIOybubMGmiRunYM8-pSaVvXfBQSkM2Eygh15tkKCDHf8X3InAkPh7HQn13mP5y1gFRLsVAUWb-91PeHASP6hluUEdsX3uLQ9OJFenKrk.0zS9Goy08bfttd4h4Jtb2JV8vbJ8-3Wb4AJWqf0eUALMxOChB3sSBKW37s4vDb1NKOnoqOeoYQUBijqRGu9YLKIAimwo7Uvl0CuD7bWNrERweBqNVWjfGhlE8Yvvklm5YhCk5XY02pOa3IjMf_TDKhbTr8bh_20SXevnDk80XKg_3mWbhuieL23kx835AokAg9JpEkgydBBqo8nzQg23R5xJzKRT64kgb5GBlzuM9Oxh7pXsmzlYf"
+ keyword = "paperform"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPaperform_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paperform secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paperform,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paperform",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paperform secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paperform,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Paperform.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Paperform.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/paralleldots/paralleldots_integration_test.go b/pkg/detectors/paralleldots/paralleldots_integration_test.go
new file mode 100644
index 000000000..e5931de4d
--- /dev/null
+++ b/pkg/detectors/paralleldots/paralleldots_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paralleldots
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestParalleldots_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PARALLELDOTS")
+ inactiveSecret := testSecrets.MustGetField("PARALLELDOTS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ParallelDots,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ParallelDots,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paralleldots.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paralleldots.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paralleldots/paralleldots_test.go b/pkg/detectors/paralleldots/paralleldots_test.go
index e5931de4d..9fc8e7844 100644
--- a/pkg/detectors/paralleldots/paralleldots_test.go
+++ b/pkg/detectors/paralleldots/paralleldots_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package paralleldots
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestParalleldots_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PARALLELDOTS")
- inactiveSecret := testSecrets.MustGetField("PARALLELDOTS_INACTIVE")
+var (
+ validPattern = "Ki82DlwEBI9dqr4k3cZPH6Z5fP39XgEPktFfgZTM4mW"
+ invalidPattern = "Ki82DlwEBI9dqr4k3cZPH?Z5fP39XgEPktFfgZTM4mW"
+ keyword = "paralleldots"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestParalleldots_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ParallelDots,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paralleldots",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paralleldots secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ParallelDots,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Paralleldots.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Paralleldots.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/parsehub/parsehub_integration_test.go b/pkg/detectors/parsehub/parsehub_integration_test.go
new file mode 100644
index 000000000..0e3cd4560
--- /dev/null
+++ b/pkg/detectors/parsehub/parsehub_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package parsehub
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestParsehub_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PARSEHUB")
+ inactiveSecret := testSecrets.MustGetField("PARSEHUB_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a parsehub secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Parsehub,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a parsehub secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Parsehub,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Parsehub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Parsehub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/parsehub/parsehub_test.go b/pkg/detectors/parsehub/parsehub_test.go
index 0e3cd4560..79ad005ed 100644
--- a/pkg/detectors/parsehub/parsehub_test.go
+++ b/pkg/detectors/parsehub/parsehub_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package parsehub
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestParsehub_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PARSEHUB")
- inactiveSecret := testSecrets.MustGetField("PARSEHUB_INACTIVE")
+var (
+ validPattern = "vZrEclFwOqeA"
+ invalidPattern = "vZrEcl?wOqeA"
+ keyword = "parsehub"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestParsehub_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a parsehub secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Parsehub,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword parsehub",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a parsehub secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Parsehub,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Parsehub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Parsehub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/parsers/parsers_integration_test.go b/pkg/detectors/parsers/parsers_integration_test.go
new file mode 100644
index 000000000..a396b9ef3
--- /dev/null
+++ b/pkg/detectors/parsers/parsers_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package parsers
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestParsers_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PARSERS")
+ inactiveSecret := testSecrets.MustGetField("PARSERS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a parsers secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Parsers,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a parsers secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Parsers,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Parsers.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Parsers.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/parsers/parsers_test.go b/pkg/detectors/parsers/parsers_test.go
index a396b9ef3..eab1af259 100644
--- a/pkg/detectors/parsers/parsers_test.go
+++ b/pkg/detectors/parsers/parsers_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package parsers
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestParsers_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PARSERS")
- inactiveSecret := testSecrets.MustGetField("PARSERS_INACTIVE")
+var (
+ validPattern = "3ui8k045w9b9n6c2fuqbr44p1fg64cjsvn2dv2uvkyxfmjler9ddsls6uqtizt7q"
+ invalidPattern = "3ui8k045w9b9n6c2fuqbr44p1fg64cjs?n2dv2uvkyxfmjler9ddsls6uqtizt7q"
+ keyword = "parsers"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestParsers_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a parsers secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Parsers,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword parsers",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a parsers secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Parsers,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Parsers.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Parsers.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/partnerstack/partnerstack_integration_test.go b/pkg/detectors/partnerstack/partnerstack_integration_test.go
new file mode 100644
index 000000000..e5bcfb777
--- /dev/null
+++ b/pkg/detectors/partnerstack/partnerstack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package partnerstack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPartnerstack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PARTNERSTACK")
+ inactiveSecret := testSecrets.MustGetField("PARTNERSTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Partnerstack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Partnerstack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Partnerstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Partnerstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/partnerstack/partnerstack_test.go b/pkg/detectors/partnerstack/partnerstack_test.go
index e5bcfb777..aea076ae1 100644
--- a/pkg/detectors/partnerstack/partnerstack_test.go
+++ b/pkg/detectors/partnerstack/partnerstack_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package partnerstack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPartnerstack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PARTNERSTACK")
- inactiveSecret := testSecrets.MustGetField("PARTNERSTACK_INACTIVE")
+var (
+ validPattern = "mDrW4HFb6fVh2ii74V41bh6EH52sWirlQuKr3svxfgGueyj32HW7OhIhNLZSgicW"
+ invalidPattern = "mDrW4HFb6fVh2ii74V41bh6EH52sWirl?uKr3svxfgGueyj32HW7OhIhNLZSgicW"
+ keyword = "partnerstack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPartnerstack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Partnerstack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword partnerstack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a partnerstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Partnerstack,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Partnerstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Partnerstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pastebin/pastebin_integration_test.go b/pkg/detectors/pastebin/pastebin_integration_test.go
new file mode 100644
index 000000000..e7aca2670
--- /dev/null
+++ b/pkg/detectors/pastebin/pastebin_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pastebin
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPastebin_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PASTEBIN")
+ inactiveSecret := testSecrets.MustGetField("PASTEBIN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pastebin secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pastebin,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pastebin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pastebin,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pastebin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pastebin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pastebin/pastebin_test.go b/pkg/detectors/pastebin/pastebin_test.go
index e7aca2670..5c1006ef9 100644
--- a/pkg/detectors/pastebin/pastebin_test.go
+++ b/pkg/detectors/pastebin/pastebin_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pastebin
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPastebin_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PASTEBIN")
- inactiveSecret := testSecrets.MustGetField("PASTEBIN_INACTIVE")
+var (
+ validPattern = "92q9vNtJQRsjQDdbiYRRHBUNBYHSymKL"
+ invalidPattern = "92q9vNtJQRsjQDdb?YRRHBUNBYHSymKL"
+ keyword = "pastebin"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPastebin_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pastebin secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pastebin,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pastebin",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pastebin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pastebin,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pastebin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pastebin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/paydirtapp/paydirtapp_integration_test.go b/pkg/detectors/paydirtapp/paydirtapp_integration_test.go
new file mode 100644
index 000000000..950cab1a1
--- /dev/null
+++ b/pkg/detectors/paydirtapp/paydirtapp_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paydirtapp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaydirtyapp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAYDIRTAPP")
+ inactiveSecret := testSecrets.MustGetField("PAYDIRTAPP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paydirtapp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paydirtapp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paydirtyapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paydirtyapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paydirtapp/paydirtapp_test.go b/pkg/detectors/paydirtapp/paydirtapp_test.go
index 950cab1a1..ef44c0e44 100644
--- a/pkg/detectors/paydirtapp/paydirtapp_test.go
+++ b/pkg/detectors/paydirtapp/paydirtapp_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package paydirtapp
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPaydirtyapp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAYDIRTAPP")
- inactiveSecret := testSecrets.MustGetField("PAYDIRTAPP_INACTIVE")
+var (
+ validPattern = "teboslrqf1lirq5gx0ca6544y3tkzsq0"
+ invalidPattern = "teboslrqf1lirq5g?0ca6544y3tkzsq0"
+ keyword = "paydirtapp"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPaydirtyapp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paydirtapp,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paydirtapp",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paydirtapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paydirtapp,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Paydirtyapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Paydirtyapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/paymoapp/paymoapp_integration_test.go b/pkg/detectors/paymoapp/paymoapp_integration_test.go
new file mode 100644
index 000000000..371bda62b
--- /dev/null
+++ b/pkg/detectors/paymoapp/paymoapp_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paymoapp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaymoapp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAYMOAPP")
+ inactiveSecret := testSecrets.MustGetField("PAYMOAPP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paymoapp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paymoapp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paymoapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paymoapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paymoapp/paymoapp_test.go b/pkg/detectors/paymoapp/paymoapp_test.go
index 371bda62b..6886a656f 100644
--- a/pkg/detectors/paymoapp/paymoapp_test.go
+++ b/pkg/detectors/paymoapp/paymoapp_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package paymoapp
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPaymoapp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAYMOAPP")
- inactiveSecret := testSecrets.MustGetField("PAYMOAPP_INACTIVE")
+var (
+ validPattern = "EpKoocVnygMTgD6XB0c4QQ2mYIqXlt1efBuIvuihzKjU"
+ invalidPattern = "EpKoocVnygMTgD6XB0c4QQ?mYIqXlt1efBuIvuihzKjU"
+ keyword = "paymoapp"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPaymoapp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paymoapp,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paymoapp",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paymoapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paymoapp,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Paymoapp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Paymoapp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/paymongo/paymongo_integration_test.go b/pkg/detectors/paymongo/paymongo_integration_test.go
new file mode 100644
index 000000000..cd7daf81e
--- /dev/null
+++ b/pkg/detectors/paymongo/paymongo_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paymongo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaymongo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAYMONGO")
+ inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paymongo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paymongo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paymongo/paymongo_test.go b/pkg/detectors/paymongo/paymongo_test.go
index cd7daf81e..864918d4d 100644
--- a/pkg/detectors/paymongo/paymongo_test.go
+++ b/pkg/detectors/paymongo/paymongo_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package paymongo
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPaymongo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAYMONGO")
- inactiveSecret := testSecrets.MustGetField("PAYMONGO_INACTIVE")
+var (
+ validPattern = "okrrdIMzwSl350mcZkrtMkzEJ_xuLaJc"
+ invalidPattern = "okrrdIMzwSl350mc?krtMkzEJ_xuLaJc"
+ keyword = "paymongo"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPaymongo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paymongo secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paymongo,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paymongo",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paymongo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paymongo,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Paymongo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Paymongo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/paypaloauth/paypaloauth_integration_test.go b/pkg/detectors/paypaloauth/paypaloauth_integration_test.go
new file mode 100644
index 000000000..f2c8e0263
--- /dev/null
+++ b/pkg/detectors/paypaloauth/paypaloauth_integration_test.go
@@ -0,0 +1,152 @@
+//go:build detectors
+// +build detectors
+
+package paypaloauth
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaypalOauth_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAYPALOAUTH_SECRET")
+ inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
+ id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
+
+ newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
+ newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PaypalOauth,
+ Verified: true,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_PaypalOauth,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PaypalOauth,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_PaypalOauth,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "new format, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PaypalOauth,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_PaypalOauth,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PaypalOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PaypalOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paypaloauth/paypaloauth_test.go b/pkg/detectors/paypaloauth/paypaloauth_test.go
index f2c8e0263..18e52675b 100644
--- a/pkg/detectors/paypaloauth/paypaloauth_test.go
+++ b/pkg/detectors/paypaloauth/paypaloauth_test.go
@@ -1,152 +1,83 @@
-//go:build detectors
-// +build detectors
-
package paypaloauth
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPaypalOauth_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAYPALOAUTH_SECRET")
- inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE")
- id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID")
-
- newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID")
- newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET")
+var (
+ validId = "Dddpt-5MySAKlcPX07LKjhzbHTbf9m2Xv9OFSw0bTdrZ"
+ invalidId = "Dddpt-5MySAKlcPX07LKjh?bHTbf9m2Xv9OFSw0bTdrZ"
+ validKey = "PDMDWZ3BAwfXHoU.P.pJVUPIYlkJ2jVP-ns2icvSlWeb-_0Qa5"
+ invalidKey = "PDMDWZ3BAwfXHoU.P.pJVUPIY?kJ2jVP-ns2icvSlWeb-_0Qa5"
+ keyword = "paypaloauth"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPaypalOauth_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PaypalOauth,
- Verified: true,
- },
- {
- DetectorType: detectorspb.DetectorType_PaypalOauth,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PaypalOauth,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_PaypalOauth,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paypaloauth",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validKey),
+ want: []string{validId, validKey},
},
{
- name: "new format, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PaypalOauth,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_PaypalOauth,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidKey),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PaypalOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PaypalOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/paystack/paystack_integration_test.go b/pkg/detectors/paystack/paystack_integration_test.go
new file mode 100644
index 000000000..19d5531ae
--- /dev/null
+++ b/pkg/detectors/paystack/paystack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package paystack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPaystack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PAYSTACK_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("PAYSTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paystack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paystack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a paystack secret %s within but unverified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Paystack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Paystack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Paystack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/paystack/paystack_test.go b/pkg/detectors/paystack/paystack_test.go
index 19d5531ae..41b42d1f0 100644
--- a/pkg/detectors/paystack/paystack_test.go
+++ b/pkg/detectors/paystack/paystack_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package paystack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPaystack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PAYSTACK_TOKEN")
- inactiveSecret := testSecrets.MustGetField("PAYSTACK_INACTIVE")
+var (
+ validPattern = "sk_xigrvarm_cJHGpWQwCTHajG2A2o8eC8TQaQGZdoMVhgXUA9Lm"
+ invalidPattern = "sk_xigrvarm_cJHGpWQwCTHajG?A2o8eC8TQaQGZdoMVhgXUA9Lm"
+ keyword = "paystack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPaystack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paystack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paystack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword paystack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a paystack secret %s within but unverified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Paystack,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Paystack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Paystack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pdflayer/pdflayer_integration_test.go b/pkg/detectors/pdflayer/pdflayer_integration_test.go
new file mode 100644
index 000000000..d803ff921
--- /dev/null
+++ b/pkg/detectors/pdflayer/pdflayer_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pdflayer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPdfLayer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PDFLAYER")
+ inactiveSecret := testSecrets.MustGetField("PDFLAYER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PdfLayer,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PdfLayer,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PdfLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PdfLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pdflayer/pdflayer_test.go b/pkg/detectors/pdflayer/pdflayer_test.go
index d803ff921..7896435b1 100644
--- a/pkg/detectors/pdflayer/pdflayer_test.go
+++ b/pkg/detectors/pdflayer/pdflayer_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pdflayer
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPdfLayer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PDFLAYER")
- inactiveSecret := testSecrets.MustGetField("PDFLAYER_INACTIVE")
+var (
+ validPattern = "wv8d7n0j02zxzrsbcs8cyk9yvqch9gvr"
+ invalidPattern = "wv8d7n0j02zxzrsb?s8cyk9yvqch9gvr"
+ keyword = "pdflayer"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPdfLayer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PdfLayer,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pdflayer",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pdflayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PdfLayer,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PdfLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PdfLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pdfshift/pdfshift_integration_test.go b/pkg/detectors/pdfshift/pdfshift_integration_test.go
new file mode 100644
index 000000000..bbddc1251
--- /dev/null
+++ b/pkg/detectors/pdfshift/pdfshift_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pdfshift
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPdfShift_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PDFSHIFT")
+ inactiveSecret := testSecrets.MustGetField("PDFSHIFT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PdfShift,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PdfShift,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PdfShift.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PdfShift.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pdfshift/pdfshift_test.go b/pkg/detectors/pdfshift/pdfshift_test.go
index bbddc1251..f747d8d1b 100644
--- a/pkg/detectors/pdfshift/pdfshift_test.go
+++ b/pkg/detectors/pdfshift/pdfshift_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pdfshift
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPdfShift_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PDFSHIFT")
- inactiveSecret := testSecrets.MustGetField("PDFSHIFT_INACTIVE")
+var (
+ validPattern = "b03ed226557f474fda3f5a8fdd498f7c"
+ invalidPattern = "b03e?226557f474fda3f5a8fdd498f7c"
+ keyword = "pdfshift"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPdfShift_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PdfShift,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pdfshift",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pdfshift secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PdfShift,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PdfShift.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PdfShift.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/peopledatalabs/peopledatalabs_integration_test.go b/pkg/detectors/peopledatalabs/peopledatalabs_integration_test.go
new file mode 100644
index 000000000..c913f7f04
--- /dev/null
+++ b/pkg/detectors/peopledatalabs/peopledatalabs_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package peopledatalabs
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPeopleDataLabs_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PEOPLEDATALABS")
+ inactiveSecret := testSecrets.MustGetField("PEOPLEDATALABS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PeopleDataLabs,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PeopleDataLabs,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PeopleDataLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PeopleDataLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/peopledatalabs/peopledatalabs_test.go b/pkg/detectors/peopledatalabs/peopledatalabs_test.go
index c913f7f04..25523d477 100644
--- a/pkg/detectors/peopledatalabs/peopledatalabs_test.go
+++ b/pkg/detectors/peopledatalabs/peopledatalabs_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package peopledatalabs
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPeopleDataLabs_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PEOPLEDATALABS")
- inactiveSecret := testSecrets.MustGetField("PEOPLEDATALABS_INACTIVE")
+var (
+ validPattern = "tp8k005gi460y93zclyrifwjnij6492iiwignkeuccqjkrs1rqcw0lsuvjm39ij9"
+ invalidPattern = "tp8k005g?460y93zclyrifwjnij6492iiwignkeuccqjkrs1rqcw0lsuvjm39ij9"
+ keyword = "peopledatalabs"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPeopleDataLabs_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PeopleDataLabs,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword peopledatalabs",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a peopledatalabs secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PeopleDataLabs,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PeopleDataLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PeopleDataLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pepipost/pepipost_integration_test.go b/pkg/detectors/pepipost/pepipost_integration_test.go
new file mode 100644
index 000000000..ced4dc843
--- /dev/null
+++ b/pkg/detectors/pepipost/pepipost_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pepipost
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPepipost_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PEPIPOST_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("PEPIPOST_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pepipost secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pepipost,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pepipost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pepipost,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pepipost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pepipost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pepipost/pepipost_test.go b/pkg/detectors/pepipost/pepipost_test.go
index ced4dc843..0d27ee25b 100644
--- a/pkg/detectors/pepipost/pepipost_test.go
+++ b/pkg/detectors/pepipost/pepipost_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pepipost
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPepipost_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PEPIPOST_TOKEN")
- inactiveSecret := testSecrets.MustGetField("PEPIPOST_INACTIVE")
+var (
+ validPattern = "zajmEjyCrhFdm1gQXoZD6r8WKLubC-B2"
+ invalidPattern = "zajmEjyCrhFdm1gQ?oZD6r8WKLubC-B2"
+ keyword = "pepipost"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPepipost_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pepipost secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pepipost,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pepipost",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pepipost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pepipost,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pepipost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pepipost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/percy/percy_integration_test.go b/pkg/detectors/percy/percy_integration_test.go
new file mode 100644
index 000000000..aad571e5b
--- /dev/null
+++ b/pkg/detectors/percy/percy_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package percy
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPercy_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PERCY_VERIFIED")
+ inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Percy,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Percy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/percy/percy_test.go b/pkg/detectors/percy/percy_test.go
index aad571e5b..8942cbedb 100644
--- a/pkg/detectors/percy/percy_test.go
+++ b/pkg/detectors/percy/percy_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package percy
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPercy_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PERCY_VERIFIED")
- inactiveSecret := testSecrets.MustGetField("PERCY_INACTIVE")
+var (
+ validPattern = "84f2cfA002913e5afbe0a43d71e49ac9389Ab4f4f827bceAed69ec34f844ed22"
+ invalidPattern = "84f2cfA002913?5afbe0a43d71e49ac9389Ab4f4f827bceAed69ec34f844ed22"
+ keyword = "percy"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPercy_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a percy secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Percy,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword percy",
+ input: fmt.Sprintf("%s token = 'PERCY_TOKEN=%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a percy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Percy,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = 'PERCY_TOKEN=%s' | 'PERCY_TOKEN=%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = 'PERCY_TOKEN=%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = 'PERCY_TOKEN=%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Percy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Percy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pinata/pinata_integration_test.go b/pkg/detectors/pinata/pinata_integration_test.go
new file mode 100644
index 000000000..9231cef77
--- /dev/null
+++ b/pkg/detectors/pinata/pinata_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package pinata
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPinata_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PINATA")
+ key := testSecrets.MustGetField("PINATA_KEY")
+ inactiveSecret := testSecrets.MustGetField("PINATA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s", secret, key)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pinata,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pinata,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pinata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pinata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pinata/pinata_test.go b/pkg/detectors/pinata/pinata_test.go
index 9231cef77..fb46c46d8 100644
--- a/pkg/detectors/pinata/pinata_test.go
+++ b/pkg/detectors/pinata/pinata_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package pinata
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPinata_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PINATA")
- key := testSecrets.MustGetField("PINATA_KEY")
- inactiveSecret := testSecrets.MustGetField("PINATA_INACTIVE")
+var (
+ validKey = "xw4xxzwjzerq6rf3zvd8zwnlh0yq62g4f7l97xxlg4u1043zrx4ndtptkoqdn49e"
+ invalidKey = "xw4xxzwjzerq6r?3zvd8zwnlh0yq62g4f7l97xxlg4u1043zrx4ndtptkoqdn49e"
+ validId = "lnurl7e15mgdgv1hyalo"
+ invalidId = "lnurl7e15m?dgv1hyalo"
+ keyword = "pinata"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPinata_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s", secret, key)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pinata,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pinata",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pinata secret %s within pinata %s but not valid", inactiveSecret, key)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pinata,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pinata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pinata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pipedream/pipedream_integration_test.go b/pkg/detectors/pipedream/pipedream_integration_test.go
new file mode 100644
index 000000000..202da5a57
--- /dev/null
+++ b/pkg/detectors/pipedream/pipedream_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package pipedream
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPipedream_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PIPEDREAM")
+ inactiveSecret := testSecrets.MustGetField("PIPEDREAM_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pipedream secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pipedream,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pipedream secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pipedream,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pipedream.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pipedream.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pipedream/pipedream_test.go b/pkg/detectors/pipedream/pipedream_test.go
index 202da5a57..d6eeab70d 100644
--- a/pkg/detectors/pipedream/pipedream_test.go
+++ b/pkg/detectors/pipedream/pipedream_test.go
@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
package pipedream
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPipedream_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PIPEDREAM")
- inactiveSecret := testSecrets.MustGetField("PIPEDREAM_INACTIVE")
+var (
+ validPattern = "dh6zkbdzrjlsz5gy4gj2zg9stqttbd65"
+ invalidPattern = "dh6zkbdzrjlsz5gy?gj2zg9stqttbd65"
+ keyword = "pipedream"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPipedream_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pipedream secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pipedream,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pipedream",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pipedream secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pipedream,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pipedream.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pipedream.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/pipedrive/pipedrive_integration_test.go b/pkg/detectors/pipedrive/pipedrive_integration_test.go
new file mode 100644
index 000000000..271a4a3f0
--- /dev/null
+++ b/pkg/detectors/pipedrive/pipedrive_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pipedrive
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPipedrive_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PIPEDRIVE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("PIPEDRIVE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pipedrive,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pipedrive,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pipedrive.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pipedrive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pipedrive/pipedrive_test.go b/pkg/detectors/pipedrive/pipedrive_test.go
index 271a4a3f0..25301e8c8 100644
--- a/pkg/detectors/pipedrive/pipedrive_test.go
+++ b/pkg/detectors/pipedrive/pipedrive_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pipedrive
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPipedrive_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PIPEDRIVE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("PIPEDRIVE_INACTIVE")
+var (
+ validPattern = "ZVt2kcilMoT5gvZDppcXkF5JPzyinJrkSwroZ9dB"
+ invalidPattern = "ZVt2?cilMoT5gvZDppcXkF5JPzyinJrkSwroZ9dB"
+ keyword = "pipedrive"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPipedrive_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pipedrive,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pipedrive",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pipedrive secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pipedrive,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pipedrive.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pipedrive.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pivotaltracker/pivotaltracker_integration_test.go b/pkg/detectors/pivotaltracker/pivotaltracker_integration_test.go
new file mode 100644
index 000000000..5e2809a56
--- /dev/null
+++ b/pkg/detectors/pivotaltracker/pivotaltracker_integration_test.go
@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package pivotaltracker
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPivotalTracker_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PIVOTALTRACKER")
+ secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PivotalTracker,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PivotalTracker,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pivotaltracker/pivotaltracker_test.go b/pkg/detectors/pivotaltracker/pivotaltracker_test.go
index 5e2809a56..9962cd484 100644
--- a/pkg/detectors/pivotaltracker/pivotaltracker_test.go
+++ b/pkg/detectors/pivotaltracker/pivotaltracker_test.go
@@ -1,119 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pivotaltracker
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPivotalTracker_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PIVOTALTRACKER")
- secretInactive := testSecrets.MustGetField("PIVOTALTRACKER_INACTIVE")
+var (
+ validPattern = "hzwlc7d249nw76xwx372n09spjhjgidv"
+ invalidPattern = "hzwlc7d249nw76?wx372n09spjhjgidv"
+ keyword = "pivotaltracker"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPivotalTracker_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PivotalTracker,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pivotaltracker",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pivotal secret %s within", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PivotalTracker,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PivotalTracker.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PivotalTracker.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pixabay/pixabay_integration_test.go b/pkg/detectors/pixabay/pixabay_integration_test.go
new file mode 100644
index 000000000..62c962ec3
--- /dev/null
+++ b/pkg/detectors/pixabay/pixabay_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pixabay
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPixabay_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PIXABAY")
+ inactiveSecret := testSecrets.MustGetField("PIXABAY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pixabay secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pixabay,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pixabay secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pixabay,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pixabay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pixabay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pixabay/pixabay_test.go b/pkg/detectors/pixabay/pixabay_test.go
index 62c962ec3..72673940f 100644
--- a/pkg/detectors/pixabay/pixabay_test.go
+++ b/pkg/detectors/pixabay/pixabay_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pixabay
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPixabay_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PIXABAY")
- inactiveSecret := testSecrets.MustGetField("PIXABAY_INACTIVE")
+var (
+ validPattern = "yz4gjwgew94zn73y9ds7hlwob6ytl70827"
+ invalidPattern = "yz4gjwgew94zn73y9?s7hlwob6ytl70827"
+ keyword = "pixabay"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPixabay_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pixabay secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pixabay,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pixabay",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pixabay secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pixabay,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pixabay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pixabay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/plaidkey/plaidkey_integration_test.go b/pkg/detectors/plaidkey/plaidkey_integration_test.go
new file mode 100644
index 000000000..624c22e2f
--- /dev/null
+++ b/pkg/detectors/plaidkey/plaidkey_integration_test.go
@@ -0,0 +1,126 @@
+//go:build detectors
+// +build detectors
+
+package plaidkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlaidKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
+ inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
+ id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
+ // env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
+ env := "development"
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within plaidkey %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlaidKey,
+ Verified: true,
+ ExtraData: map[string]string{
+ "environment": fmt.Sprintf("https://%s.plaid.com", env),
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within but plaidkey %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlaidKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PlaidKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PlaidKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/plaidkey/plaidkey_test.go b/pkg/detectors/plaidkey/plaidkey_test.go
index 624c22e2f..73241d75a 100644
--- a/pkg/detectors/plaidkey/plaidkey_test.go
+++ b/pkg/detectors/plaidkey/plaidkey_test.go
@@ -1,126 +1,83 @@
-//go:build detectors
-// +build detectors
-
package plaidkey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPlaidKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
- inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
- id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
- // env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
- env := "development"
+var (
+ validKey = "3vl81ihtozf9im7kqz7ldp6kxbsd8y"
+ invalidKey = "3vl81ihtozf9im7?qz7ldp6kxbsd8y"
+ validId = "ic1mh5b49ycvmz2vgvlgxtb0"
+ invalidId = "ic1?h5b49ycvmz2vgvlgxtb0"
+ keyword = "plaid"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPlaidKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within plaidkey %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlaidKey,
- Verified: true,
- ExtraData: map[string]string{
- "environment": fmt.Sprintf("https://%s.plaid.com", env),
- },
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword plaid",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a plaidkey secret %s within but plaidkey %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlaidKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PlaidKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PlaidKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/planetscale/planetscale_integration_test.go b/pkg/detectors/planetscale/planetscale_integration_test.go
new file mode 100644
index 000000000..315f8f97b
--- /dev/null
+++ b/pkg/detectors/planetscale/planetscale_integration_test.go
@@ -0,0 +1,164 @@
+//go:build detectors
+// +build detectors
+
+package planetscale
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanetscale_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PLANET_SCALE_TOKEN")
+ secretID := testSecrets.MustGetField("PLANET_SCALE_ID")
+ inactiveSecret := testSecrets.MustGetField("PLANET_SCALE_TOKEN_INACTIVE")
+ inactiveSecretID := testSecrets.MustGetField("PLANET_SCALE_ID_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanetScale,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s but not valid", inactiveSecret, inactiveSecretID)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanetScale,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanetScale,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanetScale,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Planetscale.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Planetscale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/planetscale/planetscale_test.go b/pkg/detectors/planetscale/planetscale_test.go
index 315f8f97b..da30c0d85 100644
--- a/pkg/detectors/planetscale/planetscale_test.go
+++ b/pkg/detectors/planetscale/planetscale_test.go
@@ -1,163 +1,82 @@
-//go:build detectors
-// +build detectors
-
package planetscale
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPlanetscale_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PLANET_SCALE_TOKEN")
- secretID := testSecrets.MustGetField("PLANET_SCALE_ID")
- inactiveSecret := testSecrets.MustGetField("PLANET_SCALE_TOKEN_INACTIVE")
- inactiveSecretID := testSecrets.MustGetField("PLANET_SCALE_ID_INACTIVE")
+var (
+ validUsername = "fo57eya1lvvh"
+ invalidUsername = "fo57ey?1lvvh"
+ validPassword = "pscale_tkn_toLyJg1LfgY8vDQr9vhfF2HcT410U3q1alxt7012kcV"
+ invalidPassword = "pscale_tkn_toLyJg1LfgY8vDQr?vhfF2HcT410U3q1alxt7012kcV"
+ keyword = "planetscale"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPlanetscale_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanetScale,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s but not valid", inactiveSecret, inactiveSecretID)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanetScale,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword planetscale",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validPassword),
+ want: []string{validUsername + ":" + validPassword},
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanetScale,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planetscale secret %s within with id %s", secret, secretID)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanetScale,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidPassword),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Planetscale.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Planetscale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/planetscaledb/planetscaledb_integration_test.go b/pkg/detectors/planetscaledb/planetscaledb_integration_test.go
new file mode 100644
index 000000000..57a1ae3c4
--- /dev/null
+++ b/pkg/detectors/planetscaledb/planetscaledb_integration_test.go
@@ -0,0 +1,129 @@
+//go:build detectors
+// +build detectors
+
+package planetscaledb
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanetscaledb_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ username := testSecrets.MustGetField("PLANET_SCALEDB_USERNAME")
+ host := testSecrets.MustGetField("PLANET_SCALEDB_HOST")
+ password := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD")
+ inactivePassword := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, password, host)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanetScaleDb,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, inactivePassword, host)),
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanetScaleDb,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Planetscaledb.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Planetscaledb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/planetscaledb/planetscaledb_test.go b/pkg/detectors/planetscaledb/planetscaledb_test.go
index 57a1ae3c4..a1a0fc223 100644
--- a/pkg/detectors/planetscaledb/planetscaledb_test.go
+++ b/pkg/detectors/planetscaledb/planetscaledb_test.go
@@ -1,128 +1,84 @@
-//go:build detectors
-// +build detectors
-
package planetscaledb
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPlanetscaledb_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- username := testSecrets.MustGetField("PLANET_SCALEDB_USERNAME")
- host := testSecrets.MustGetField("PLANET_SCALEDB_HOST")
- password := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD")
- inactivePassword := testSecrets.MustGetField("PLANET_SCALEDB_PASSWORD_INACTIVE")
+var (
+ validUsername = "82mygyuh2y23aw1k8lzv"
+ invalidUsername = "8?mygyuh2y23aw1k8lzv"
+ validPassword = "pscale_pw_iAhKQKjU8nUrHagaygAubhM6x0LTaBz6kOyqx9AIS6V"
+ invalidPassword = "pscale_pw_iAhKQKjU8nUrHaga?gAubhM6x0LTaBz6kOyqx9AIS6V"
+ validHost = "gcp.connect.psdb.cloud"
+ invalidHost = "gcp?connect.psdb.cloud"
+ keyword = "planetscaledb"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPlanetscaledb_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, password, host)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanetScaleDb,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword planetscaledb",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validPassword, keyword, validHost),
+ want: []string{validHost + "\t" + validUsername + "\t" + validPassword},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planetscaledb secret %s %s %s", username, inactivePassword, host)),
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanetScaleDb,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidPassword, keyword, invalidHost),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Planetscaledb.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Planetscaledb.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/planviewleankit/planviewleankit_integration_test.go b/pkg/detectors/planviewleankit/planviewleankit_integration_test.go
new file mode 100644
index 000000000..99a39767f
--- /dev/null
+++ b/pkg/detectors/planviewleankit/planviewleankit_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package planviewleankit
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanviewLeanKit_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PLANVIEWLEANKIT")
+ inactiveSecret := testSecrets.MustGetField("PLANVIEWLEANKIT_INACTIVE")
+ subdomain := testSecrets.MustGetField("PLANVIEWLEANKIT_SUBDOMAIN")
+
+ // log.Println(secret)
+ // log.Println(inactiveSecret)
+ // log.Println(subdomain)
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within", subdomain, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within but not valid", subdomain, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PlanviewLeanKit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PlanviewLeanKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/planviewleankit/planviewleankit_test.go b/pkg/detectors/planviewleankit/planviewleankit_test.go
index 99a39767f..bb6cb22d9 100644
--- a/pkg/detectors/planviewleankit/planviewleankit_test.go
+++ b/pkg/detectors/planviewleankit/planviewleankit_test.go
@@ -1,121 +1,82 @@
-//go:build detectors
-// +build detectors
-
package planviewleankit
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPlanviewLeanKit_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PLANVIEWLEANKIT")
- inactiveSecret := testSecrets.MustGetField("PLANVIEWLEANKIT_INACTIVE")
- subdomain := testSecrets.MustGetField("PLANVIEWLEANKIT_SUBDOMAIN")
-
- // log.Println(secret)
- // log.Println(inactiveSecret)
- // log.Println(subdomain)
+var (
+ validKey = "b1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277bd5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4f"
+ invalidKey = "B1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277bd5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4F"
+ validSubDomain = "subdomain.pYwPPnHSc"
+ invalidSubDomain = "?ubdomain.pYwPPnHS?"
+ keyword = "planviewleankit"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPlanviewLeanKit_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within", subdomain, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword planviewleankit",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSubDomain),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planviewleankit subdomain %s with planviewleankit secret %s within but not valid", subdomain, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PlanviewLeanKit,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSubDomain),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PlanviewLeanKit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PlanviewLeanKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/planyo/planyo_integration_test.go b/pkg/detectors/planyo/planyo_integration_test.go
new file mode 100644
index 000000000..03730cd2f
--- /dev/null
+++ b/pkg/detectors/planyo/planyo_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package planyo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlanyo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PLANYO")
+ inactiveSecret := testSecrets.MustGetField("PLANYO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planyo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Planyo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a planyo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Planyo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Planyo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Planyo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/planyo/planyo_test.go b/pkg/detectors/planyo/planyo_test.go
index 03730cd2f..dc0bfc3f3 100644
--- a/pkg/detectors/planyo/planyo_test.go
+++ b/pkg/detectors/planyo/planyo_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package planyo
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPlanyo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PLANYO")
- inactiveSecret := testSecrets.MustGetField("PLANYO_INACTIVE")
+var (
+ validPattern = "big57xmqvemmsopa2s7s3805oo70dzxk7subgcao8zerjg89ze8walz5si63x7"
+ invalidPattern = "big57xmqvemmsopa2s7s3805oo70dzx?7subgcao8zerjg89ze8walz5si63x7"
+ keyword = "planyo"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPlanyo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planyo secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Planyo,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword planyo",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a planyo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Planyo,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Planyo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Planyo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/plivo/plivo_integration_test.go b/pkg/detectors/plivo/plivo_integration_test.go
new file mode 100644
index 000000000..26db005f4
--- /dev/null
+++ b/pkg/detectors/plivo/plivo_integration_test.go
@@ -0,0 +1,123 @@
+//go:build detectors
+// +build detectors
+
+package plivo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPlivo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PLIVO_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("PLIVO_INACTIVE")
+ id := testSecrets.MustGetField("PLIVO_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Plivo,
+ Redacted: id,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/ but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Plivo,
+ Redacted: id,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Plivo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Plivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/plivo/plivo_test.go b/pkg/detectors/plivo/plivo_test.go
index 26db005f4..7b11b9212 100644
--- a/pkg/detectors/plivo/plivo_test.go
+++ b/pkg/detectors/plivo/plivo_test.go
@@ -1,123 +1,83 @@
-//go:build detectors
-// +build detectors
-
package plivo
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPlivo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PLIVO_TOKEN")
- inactiveSecret := testSecrets.MustGetField("PLIVO_INACTIVE")
- id := testSecrets.MustGetField("PLIVO_ID")
+var (
+ validId = "YGIQXPGZSVVGVGOREMAE"
+ invalidId = "YGIQXPGZS?VGVGOREMAE"
+ validKey = "qFj32Da8vf-g_-8qLu9P_k8XPyAHGrvKrzGTQIN4"
+ invalidKey = "qFj32Da8vf-g?-8qLu9P_k8XPyAHGrvKrzGTQIN4"
+ keyword = "plivo"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPlivo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Plivo,
- Redacted: id,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword plivo",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validKey),
+ want: []string{validKey + validId},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a plivo secret %s within https://api.plivo.com/v1/Account/%s/Number/ but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Plivo,
- Redacted: id,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidKey),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Plivo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Plivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/podio/podio_integration_test.go b/pkg/detectors/podio/podio_integration_test.go
new file mode 100644
index 000000000..b7a162f03
--- /dev/null
+++ b/pkg/detectors/podio/podio_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package podio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPodio_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PODIO")
+ inactiveSecret := testSecrets.MustGetField("PODIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a podio secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Podio,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a podio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Podio,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Podio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Podio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/podio/podio_test.go b/pkg/detectors/podio/podio_test.go
index b7a162f03..d089c4c87 100644
--- a/pkg/detectors/podio/podio_test.go
+++ b/pkg/detectors/podio/podio_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package podio
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPodio_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PODIO")
- inactiveSecret := testSecrets.MustGetField("PODIO_INACTIVE")
+var (
+ validPattern = "d3ij9dt7n9pu9enq0seldrpwcbl6p1ky"
+ invalidPattern = "d3ij9dt7n9pu9enq?seldrpwcbl6p1ky"
+ keyword = "podio"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPodio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a podio secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Podio,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword podio",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a podio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Podio,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Podio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Podio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pollsapi/pollsapi_integration_test.go b/pkg/detectors/pollsapi/pollsapi_integration_test.go
new file mode 100644
index 000000000..7e28de28d
--- /dev/null
+++ b/pkg/detectors/pollsapi/pollsapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pollsapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPollsAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POLLSAPI")
+ inactiveSecret := testSecrets.MustGetField("POLLSAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PollsAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PollsAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PollsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PollsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pollsapi/pollsapi_test.go b/pkg/detectors/pollsapi/pollsapi_test.go
index 7e28de28d..2c7dbdd20 100644
--- a/pkg/detectors/pollsapi/pollsapi_test.go
+++ b/pkg/detectors/pollsapi/pollsapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pollsapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPollsAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POLLSAPI")
- inactiveSecret := testSecrets.MustGetField("POLLSAPI_INACTIVE")
+var (
+ validPattern = "WHWPPKLOD23IRAUKKY24V9WWV6IC"
+ invalidPattern = "WHWPPKLOD23IRA?KKY24V9WWV6IC"
+ keyword = "pollsapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPollsAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PollsAPI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pollsapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pollsapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PollsAPI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PollsAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PollsAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/poloniex/poloniex_integration_test.go b/pkg/detectors/poloniex/poloniex_integration_test.go
new file mode 100644
index 000000000..22dd53d4a
--- /dev/null
+++ b/pkg/detectors/poloniex/poloniex_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package poloniex
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPoloniex_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ key := testSecrets.MustGetField("POLONIEX_KEY")
+ inactiveKey := testSecrets.MustGetField("POLONIEX_KEY_INACTIVE")
+ secret := testSecrets.MustGetField("POLONIEX")
+ inactiveSecret := testSecrets.MustGetField("POLONIEX_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within", key, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Poloniex,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Poloniex,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Poloniex.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Poloniex.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/poloniex/poloniex_test.go b/pkg/detectors/poloniex/poloniex_test.go
index 22dd53d4a..47b38fe7e 100644
--- a/pkg/detectors/poloniex/poloniex_test.go
+++ b/pkg/detectors/poloniex/poloniex_test.go
@@ -1,122 +1,83 @@
-//go:build detectors
-// +build detectors
-
package poloniex
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPoloniex_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- key := testSecrets.MustGetField("POLONIEX_KEY")
- inactiveKey := testSecrets.MustGetField("POLONIEX_KEY_INACTIVE")
- secret := testSecrets.MustGetField("POLONIEX")
- inactiveSecret := testSecrets.MustGetField("POLONIEX_INACTIVE")
+var (
+ validKey = "2ELRAUEF-NB06CX0R-ZGXL88HF-BX8B77S6"
+ invalidKey = "2ELRAUEF?NB06CX0R-ZGXL88HF-BX8B77S6"
+ validSecret = "d5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4fb1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277b"
+ invalidSecret = "D5dc2f6c299bb01895c604f7038c27af9035937823c7c0e8e25c2efe7f6ddd4fb1bb66781f857e6edf539042389e77043396b123c85197676719ef79a420277B"
+ keyword = "poloniex"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPoloniex_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within", key, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Poloniex,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword poloniex",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
+ want: []string{validKey + validSecret},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a poloniex key %s with poloniex secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Poloniex,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Poloniex.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Poloniex.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/polygon/polygon_integration_test.go b/pkg/detectors/polygon/polygon_integration_test.go
new file mode 100644
index 000000000..3565f38e4
--- /dev/null
+++ b/pkg/detectors/polygon/polygon_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package polygon
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPolygon_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POLYGON_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Polygon,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Polygon,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/polygon/polygon_test.go b/pkg/detectors/polygon/polygon_test.go
index 3565f38e4..dda41ae02 100644
--- a/pkg/detectors/polygon/polygon_test.go
+++ b/pkg/detectors/polygon/polygon_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package polygon
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPolygon_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POLYGON_TOKEN")
- inactiveSecret := testSecrets.MustGetField("POLYGON_INACTIVE")
+var (
+ validPattern = "vfINal1N4C0YH6fME3dxmRGaeuO1WjnP"
+ invalidPattern = "vfINal1N4C0YH6fM?3dxmRGaeuO1WjnP"
+ keyword = "polygon"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPolygon_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a polygon secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Polygon,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword polygon",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a polygon secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Polygon,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Polygon.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Polygon.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/portainer/portainer_integration_test.go b/pkg/detectors/portainer/portainer_integration_test.go
new file mode 100644
index 000000000..49ca42d4a
--- /dev/null
+++ b/pkg/detectors/portainer/portainer_integration_test.go
@@ -0,0 +1,131 @@
+//go:build detectors
+// +build detectors
+
+package portainer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPortainer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PORTAINER")
+ endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
+ inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Portainer,
+ Verified: true,
+ RawV2: []byte(secret + endpoint),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Portainer,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + endpoint),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/portainer/portainer_test.go b/pkg/detectors/portainer/portainer_test.go
index 49ca42d4a..56cbe91f6 100644
--- a/pkg/detectors/portainer/portainer_test.go
+++ b/pkg/detectors/portainer/portainer_test.go
@@ -1,130 +1,82 @@
-//go:build detectors
-// +build detectors
-
package portainer
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPortainer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PORTAINER")
- endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
- inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
+var (
+ validEndpoint = "http://>xC'w//b7U@CtF|>|Fqw'2Z"
+ invalidEndpoint = "?ttp://>xC'w//b7U@CtF|>|Fqw'2?"
+ validToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.EHHJE6Aht7Exhje8rUMLScr2bxcoTvWBl9bjMYZhCMYMLPD3EpUZL9SNd839DcI95lYtMfclPffpFrrJ0BbgryxnrfUSeeSKHu.W9Ur5_DLIBpXO3mfh404_7Kt9o8XZRnLTLyam2fdhB_"
+ invalidToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.EHHJE6Aht7Exhje8rUM?Scr2bxcoTvWBl9bjMYZhCMYMLPD3EpUZL9SNd839DcI95lYtMfclPffpFrrJ0BbgryxnrfUSeeSKHu.W9Ur5_DLIBpXO3mfh404_7Kt9o8XZRnLTLyam2fdhB_"
+ keyword = "portainer"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPortainer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Portainer,
- Verified: true,
- RawV2: []byte(secret + endpoint),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword portainer",
+ input: fmt.Sprintf("%s token - '%s;'\n%s token - '%s'\n", keyword, validEndpoint, keyword, validToken),
+ want: []string{validToken + validEndpoint},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Portainer,
- Verified: false,
- RawV2: []byte(inactiveSecret + endpoint),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s;'\n%s token - '%s'\n", keyword, invalidEndpoint, keyword, invalidToken),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.go
index 257f3b6c4..0f0c6fec3 100644
--- a/pkg/detectors/portainertoken/portainertoken.go
+++ b/pkg/detectors/portainertoken/portainertoken.go
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)
if err != nil {
+ fmt.Printf("\nINVALID URL\n")
// if the URL is invalid just move onto the next one
continue
}
diff --git a/pkg/detectors/portainertoken/portainertoken_integration_test.go b/pkg/detectors/portainertoken/portainertoken_integration_test.go
new file mode 100644
index 000000000..cf2c35d22
--- /dev/null
+++ b/pkg/detectors/portainertoken/portainertoken_integration_test.go
@@ -0,0 +1,131 @@
+//go:build detectors
+// +build detectors
+
+package portainertoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPortainertoken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PORTAINERTOKEN")
+ inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
+ endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PortainerToken,
+ Verified: true,
+ RawV2: []byte(secret + endpoint),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PortainerToken,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + endpoint),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/portainertoken/portainertoken_test.go b/pkg/detectors/portainertoken/portainertoken_test.go
index cf2c35d22..21b74bbef 100644
--- a/pkg/detectors/portainertoken/portainertoken_test.go
+++ b/pkg/detectors/portainertoken/portainertoken_test.go
@@ -1,130 +1,82 @@
-//go:build detectors
-// +build detectors
-
package portainertoken
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPortainertoken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PORTAINERTOKEN")
- inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
- endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
+var (
+ validKey = "ptr_9zmQMKyeMqEB_pei887xQBZGhGWm1jXCIT0gI"
+ invalidKey = "ptr_9zmQMKyeMqEB_pei?87xQBZGhGWm1jXCIT0gI"
+ validEndpoint = "http://api.SAaiuc8123.com:12345"
+ invalidEndpoint = "?ttp://api.SAaiuc8123.com:12345"
+ keyword = "portainertoken"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPortainertoken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PortainerToken,
- Verified: true,
- RawV2: []byte(secret + endpoint),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword portainertoken",
+ input: fmt.Sprintf("%s token - '%s'\nportainer token - '%s'\n", keyword, validKey, validEndpoint),
+ want: []string{validKey + validEndpoint},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PortainerToken,
- Verified: false,
- RawV2: []byte(inactiveSecret + endpoint),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\nportainer token - '%s'\n", keyword, invalidKey, invalidEndpoint),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/positionstack/positionstack_integration_test.go b/pkg/detectors/positionstack/positionstack_integration_test.go
new file mode 100644
index 000000000..f7494337b
--- /dev/null
+++ b/pkg/detectors/positionstack/positionstack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package positionstack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPositionStack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POSITIONSTACK")
+ inactiveSecret := testSecrets.MustGetField("POSITIONSTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a positionstack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PositionStack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a positionstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PositionStack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PositionStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PositionStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/positionstack/positionstack_test.go b/pkg/detectors/positionstack/positionstack_test.go
index f7494337b..b7d6219c1 100644
--- a/pkg/detectors/positionstack/positionstack_test.go
+++ b/pkg/detectors/positionstack/positionstack_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package positionstack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPositionStack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POSITIONSTACK")
- inactiveSecret := testSecrets.MustGetField("POSITIONSTACK_INACTIVE")
+var (
+ validPattern = "iUgwG0nYZt0TY1x5bfyWMJj02PhW7EGX"
+ invalidPattern = "iUgwG0nYZt0TY1x5?fyWMJj02PhW7EGX"
+ keyword = "positionstack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPositionStack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a positionstack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PositionStack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword positionstack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a positionstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PositionStack,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PositionStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PositionStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/postageapp/postageapp_integration_test.go b/pkg/detectors/postageapp/postageapp_integration_test.go
new file mode 100644
index 000000000..37e7596f7
--- /dev/null
+++ b/pkg/detectors/postageapp/postageapp_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package postageapp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostageApp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POSTAGEAPP")
+ inactiveSecret := testSecrets.MustGetField("POSTAGEAPP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postageapp secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PostageApp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postageapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PostageApp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PostageApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PostageApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/postageapp/postageapp_test.go b/pkg/detectors/postageapp/postageapp_test.go
index 37e7596f7..a3f7dc482 100644
--- a/pkg/detectors/postageapp/postageapp_test.go
+++ b/pkg/detectors/postageapp/postageapp_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package postageapp
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPostageApp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POSTAGEAPP")
- inactiveSecret := testSecrets.MustGetField("POSTAGEAPP_INACTIVE")
+var (
+ validPattern = "VJhmldVkyWB2bmLumsZzOtJKfxliCAXP"
+ invalidPattern = "VJh?ldVkyWB2bmLumsZzOtJKfxliCAXP"
+ keyword = "postageapp"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPostageApp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postageapp secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PostageApp,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword postageapp",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postageapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PostageApp,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PostageApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PostageApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/postbacks/postbacks_integration_test.go b/pkg/detectors/postbacks/postbacks_integration_test.go
new file mode 100644
index 000000000..fd032f3f1
--- /dev/null
+++ b/pkg/detectors/postbacks/postbacks_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package postbacks
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostbacks_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POSTBACKS")
+ inactiveSecret := testSecrets.MustGetField("POSTBACKS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postbacks secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postbacks,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postbacks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postbacks,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Postbacks.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Postbacks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/postbacks/postbacks_test.go b/pkg/detectors/postbacks/postbacks_test.go
index fd032f3f1..d011b068e 100644
--- a/pkg/detectors/postbacks/postbacks_test.go
+++ b/pkg/detectors/postbacks/postbacks_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package postbacks
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPostbacks_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POSTBACKS")
- inactiveSecret := testSecrets.MustGetField("POSTBACKS_INACTIVE")
+var (
+ validPattern = "64176f2e-e1da-3e29-2a08-19fa8bcb1838"
+ invalidPattern = "64176f2e?e1da-3e29-2a08-19fa8bcb1838"
+ keyword = "postbacks"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPostbacks_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postbacks secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postbacks,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword postbacks",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postbacks secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postbacks,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Postbacks.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Postbacks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/postgres/postgres_integration_test.go b/pkg/detectors/postgres/postgres_integration_test.go
new file mode 100644
index 000000000..107ec0b24
--- /dev/null
+++ b/pkg/detectors/postgres/postgres_integration_test.go
@@ -0,0 +1,406 @@
+//go:build detectors
+// +build detectors
+
+package postgres
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/lib/pq"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+var postgresDockerHash string
+
+const (
+ postgresUser = "postgres"
+ postgresPass = "23201da=b56ca236f3dc6736c0f9afad"
+ postgresHost = "localhost"
+ postgresPort = "5434" // Do not use 5433, as local dev environments can use it for other things
+
+ inactivePass = "inactive"
+ inactiveHost = "192.0.2.0"
+)
+
+func TestPostgres_FromChunk(t *testing.T) {
+ if err := startPostgres(); err != nil {
+ if exitErr, ok := err.(*exec.ExitError); ok {
+ t.Fatalf("could not start local postgres: %v w/stderr:\n%s", err, string(exitErr.Stderr))
+ } else {
+ t.Fatalf("could not start local postgres: %v", err)
+ }
+ }
+ defer stopPostgres()
+
+ // The detector is written to connect to the database 'postgres' if no explicit database is found in the candidate
+ // secret (because pq uses 'postgres' as a default if no database is specified). If the target cluster doesn't
+ // actually have a database with this name, but our credentials are good, then Postgres will give us a "missing
+ // database" error message instead of an authentication failure.
+ //
+ // Unfortunately, directly validating this in the automated tests is awkward because the docker image's POSTGRES_DB
+ // environment variable doesn't appear to work: The database created is always named 'postgres', no matter what
+ // POSTGRES_DB is set to. This means that we can't replicate a cluster that has no database named 'postgres', so we
+ // can't directly test what happens if we see one. To work around this, all the automated tests try to connect to
+ // the nonexistent database 'postgres2'. In this way, we test the logic of attempting to connect to a non-existent
+ // database, even though the test cases are the inverse of what we'd see in the wild.
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "found connection URI with ssl mode unset, verified",
+ s: Scanner{detectLoopback: true},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: true,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "<unset>"},
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found connection URI with ssl mode 'prefer', verified",
+ s: Scanner{detectLoopback: true},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=prefer`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: true,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "prefer"},
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found connection URI with ssl mode 'allow', verified",
+ s: Scanner{detectLoopback: true},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=allow`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: true,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "allow"},
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found connection URI with requiressl=0, verified",
+ s: Scanner{detectLoopback: true},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=0`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: true,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "prefer"},
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found connection URI without database, verified",
+ s: Scanner{detectLoopback: true},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: true,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "<unset>"},
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found connection URI, unverified",
+ s: Scanner{detectLoopback: true},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, inactivePass, postgresHost, postgresPort)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: false,
+ Raw: []byte("postgresql://postgres:inactive@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:inactive@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "<unset>"},
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "ignored localhost",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "localhost", postgresPort)),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "ignored 127.0.0.1",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "127.0.0.1", postgresPort)),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "found connection URI, unverified due to error - inactive host",
+ s: Scanner{},
+ args: func() args {
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ return args{
+ ctx: ctx,
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, inactiveHost, postgresPort)),
+ verify: true,
+ }
+ }(),
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: false,
+ Raw: []byte("postgresql://postgres:[email protected]:5434"),
+ RawV2: []byte("postgresql://postgres:[email protected]:5434"),
+ ExtraData: map[string]string{"sslmode": "<unset>"},
+ }
+ r.SetVerificationError(errors.New("i/o timeout"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found connection URI, unverified due to error - wrong port",
+ s: Scanner{detectLoopback: true},
+ args: func() args {
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ return args{
+ ctx: ctx,
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s/postgres2`, postgresUser, postgresPass, postgresHost)),
+ verify: true,
+ }
+ }(),
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: false,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
+ ExtraData: map[string]string{"sslmode": "<unset>"},
+ }
+ r.SetVerificationError(errors.New("connection refused"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found connection URI, unverified due to error - ssl not supported (using sslmode)",
+ s: Scanner{detectLoopback: true},
+ args: func() args {
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ return args{
+ ctx: ctx,
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=require`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ }
+ }(),
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: false,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "require"},
+ }
+ r.SetVerificationError(pq.ErrSSLNotSupported)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found connection URI, unverified due to error - ssl not supported (using requiressl)",
+ s: Scanner{detectLoopback: true},
+ args: func() args {
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+ return args{
+ ctx: ctx,
+ data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=1`, postgresUser, postgresPass, postgresHost, postgresPort)),
+ verify: true,
+ }
+ }(),
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Postgres,
+ Verified: false,
+ Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
+ ExtraData: map[string]string{"sslmode": "require"},
+ }
+ r.SetVerificationError(pq.ErrSSLNotSupported)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "AnalysisInfo")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func dockerLogLine(hash string, needle string) chan struct{} {
+ ch := make(chan struct{}, 1)
+ go func() {
+ for {
+ out, err := exec.Command("docker", "logs", hash).CombinedOutput()
+ if err != nil {
+ panic(err)
+ }
+ if strings.Contains(string(out), needle) {
+ ch <- struct{}{}
+ return
+ }
+ time.Sleep(1 * time.Second)
+ }
+ }()
+ return ch
+}
+
+func startPostgres() error {
+ cmd := exec.Command(
+ "docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,
+ "-e", "POSTGRES_PASSWORD="+postgresPass,
+ "-e", "POSTGRES_USER="+postgresUser,
+ "-d", "postgres",
+ )
+ fmt.Println(cmd.String())
+ out, err := cmd.Output()
+ if err != nil {
+ return err
+ }
+ postgresDockerHash = string(bytes.TrimSpace(out))
+ select {
+ case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
+ return nil
+ case <-time.After(30 * time.Second):
+ stopPostgres()
+ return errors.New("timeout waiting for postgres database to be ready")
+ }
+}
+
+func stopPostgres() {
+ exec.Command("docker", "kill", postgresDockerHash).Run()
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go
index 107ec0b24..0570cca9a 100644
--- a/pkg/detectors/postgres/postgres_test.go
+++ b/pkg/detectors/postgres/postgres_test.go
@@ -1,406 +1,83 @@
-//go:build detectors
-// +build detectors
-
package postgres
import (
- "bytes"
"context"
- "errors"
"fmt"
- "os/exec"
- "strings"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/lib/pq"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-var postgresDockerHash string
-
-const (
- postgresUser = "postgres"
- postgresPass = "23201da=b56ca236f3dc6736c0f9afad"
- postgresHost = "localhost"
- postgresPort = "5434" // Do not use 5433, as local dev environments can use it for other things
-
- inactivePass = "inactive"
- inactiveHost = "192.0.2.0"
+var (
+ validUriPattern = "postgres://sN19x:[email protected]:5432"
+ invalidUriPattern = "?ostgres://sN19x:[email protected]:5432"
+ validConnStrPartPattern = "gVmMTdkwLwmZljcIOXhEmuZ='.jD#=-;|9tD!r^6('"
+ invalidConnStrPartPattern = "gVmMTdkwLwmZljcIOXhEmu?='.jD#=-;|9tD!r^6('"
+ keyword = "postgres"
)
-func TestPostgres_FromChunk(t *testing.T) {
- if err := startPostgres(); err != nil {
- if exitErr, ok := err.(*exec.ExitError); ok {
- t.Fatalf("could not start local postgres: %v w/stderr:\n%s", err, string(exitErr.Stderr))
- } else {
- t.Fatalf("could not start local postgres: %v", err)
- }
- }
- defer stopPostgres()
-
- // The detector is written to connect to the database 'postgres' if no explicit database is found in the candidate
- // secret (because pq uses 'postgres' as a default if no database is specified). If the target cluster doesn't
- // actually have a database with this name, but our credentials are good, then Postgres will give us a "missing
- // database" error message instead of an authentication failure.
- //
- // Unfortunately, directly validating this in the automated tests is awkward because the docker image's POSTGRES_DB
- // environment variable doesn't appear to work: The database created is always named 'postgres', no matter what
- // POSTGRES_DB is set to. This means that we can't replicate a cluster that has no database named 'postgres', so we
- // can't directly test what happens if we see one. To work around this, all the automated tests try to connect to
- // the nonexistent database 'postgres2'. In this way, we test the logic of attempting to connect to a non-existent
- // database, even though the test cases are the inverse of what we'd see in the wild.
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPostgres_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- {
- name: "found connection URI with ssl mode unset, verified",
- s: Scanner{detectLoopback: true},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: true,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "<unset>"},
- },
- },
- wantErr: false,
- },
- {
- name: "found connection URI with ssl mode 'prefer', verified",
- s: Scanner{detectLoopback: true},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=prefer`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: true,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "prefer"},
- },
- },
- wantErr: false,
- },
- {
- name: "found connection URI with ssl mode 'allow', verified",
- s: Scanner{detectLoopback: true},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=allow`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: true,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "allow"},
- },
- },
- wantErr: false,
- },
- {
- name: "found connection URI with requiressl=0, verified",
- s: Scanner{detectLoopback: true},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=0`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: true,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "prefer"},
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword postgres",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validUriPattern, keyword, validConnStrPartPattern),
+ want: []string{validUriPattern},
},
{
- name: "found connection URI without database, verified",
- s: Scanner{detectLoopback: true},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: true,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "<unset>"},
- },
- },
- wantErr: false,
- },
- {
- name: "found connection URI, unverified",
- s: Scanner{detectLoopback: true},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, inactivePass, postgresHost, postgresPort)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: false,
- Raw: []byte("postgresql://postgres:inactive@localhost:5434"),
- RawV2: []byte("postgresql://postgres:inactive@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "<unset>"},
- },
- },
- wantErr: false,
- },
- {
- name: "ignored localhost",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "localhost", postgresPort)),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- {
- name: "ignored 127.0.0.1",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, "127.0.0.1", postgresPort)),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- {
- name: "found connection URI, unverified due to error - inactive host",
- s: Scanner{},
- args: func() args {
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
- defer cancel()
- return args{
- ctx: ctx,
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2`, postgresUser, postgresPass, inactiveHost, postgresPort)),
- verify: true,
- }
- }(),
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: false,
- Raw: []byte("postgresql://postgres:[email protected]:5434"),
- RawV2: []byte("postgresql://postgres:[email protected]:5434"),
- ExtraData: map[string]string{"sslmode": "<unset>"},
- }
- r.SetVerificationError(errors.New("i/o timeout"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found connection URI, unverified due to error - wrong port",
- s: Scanner{detectLoopback: true},
- args: func() args {
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
- defer cancel()
- return args{
- ctx: ctx,
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s/postgres2`, postgresUser, postgresPass, postgresHost)),
- verify: true,
- }
- }(),
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: false,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5432"),
- ExtraData: map[string]string{"sslmode": "<unset>"},
- }
- r.SetVerificationError(errors.New("connection refused"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found connection URI, unverified due to error - ssl not supported (using sslmode)",
- s: Scanner{detectLoopback: true},
- args: func() args {
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
- defer cancel()
- return args{
- ctx: ctx,
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?sslmode=require`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- }
- }(),
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: false,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "require"},
- }
- r.SetVerificationError(pq.ErrSSLNotSupported)
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found connection URI, unverified due to error - ssl not supported (using requiressl)",
- s: Scanner{detectLoopback: true},
- args: func() args {
- ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
- defer cancel()
- return args{
- ctx: ctx,
- data: []byte(fmt.Sprintf(`postgresql://%s:%s@%s:%s/postgres2?requiressl=1`, postgresUser, postgresPass, postgresHost, postgresPort)),
- verify: true,
- }
- }(),
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Postgres,
- Verified: false,
- Raw: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- RawV2: []byte("postgresql://postgres:23201da=b56ca236f3dc6736c0f9afad@localhost:5434"),
- ExtraData: map[string]string{"sslmode": "require"},
- }
- r.SetVerificationError(pq.ErrSSLNotSupported)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidUriPattern, keyword, invalidConnStrPartPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
- }
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
- }
- }
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "AnalysisInfo")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-func dockerLogLine(hash string, needle string) chan struct{} {
- ch := make(chan struct{}, 1)
- go func() {
- for {
- out, err := exec.Command("docker", "logs", hash).CombinedOutput()
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
if err != nil {
- panic(err)
- }
- if strings.Contains(string(out), needle) {
- ch <- struct{}{}
+ t.Errorf("error = %v", err)
return
}
- time.Sleep(1 * time.Second)
- }
- }()
- return ch
-}
-
-func startPostgres() error {
- cmd := exec.Command(
- "docker", "run", "--rm", "-p", postgresPort+":"+defaultPort,
- "-e", "POSTGRES_PASSWORD="+postgresPass,
- "-e", "POSTGRES_USER="+postgresUser,
- "-d", "postgres",
- )
- fmt.Println(cmd.String())
- out, err := cmd.Output()
- if err != nil {
- return err
- }
- postgresDockerHash = string(bytes.TrimSpace(out))
- select {
- case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
- return nil
- case <-time.After(30 * time.Second):
- stopPostgres()
- return errors.New("timeout waiting for postgres database to be ready")
- }
-}
-func stopPostgres() {
- exec.Command("docker", "kill", postgresDockerHash).Run()
-}
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/posthog/posthog_integration_test.go b/pkg/detectors/posthog/posthog_integration_test.go
new file mode 100644
index 000000000..70091659e
--- /dev/null
+++ b/pkg/detectors/posthog/posthog_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package posthog
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppPosthog_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPPOSTHOG_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("APPPOSTHOG_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appposthog secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PosthogApp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appposthog secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PosthogApp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AppPosthog.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AppPosthog.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/posthog/posthog_test.go b/pkg/detectors/posthog/posthog_test.go
index 70091659e..923bbc170 100644
--- a/pkg/detectors/posthog/posthog_test.go
+++ b/pkg/detectors/posthog/posthog_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package posthog
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAppPosthog_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPPOSTHOG_TOKEN")
- inactiveSecret := testSecrets.MustGetField("APPPOSTHOG_INACTIVE")
+var (
+ validPattern = "phx_C1rP9fnAtEFJvb0IYCFdeQhar2WdwUFBYHJym1F_Zqr"
+ invalidPattern = "phx_C1rP9fnAtEFJvb0IYCF?eQhar2WdwUFBYHJym1F_Zqr"
+ keyword = "posthog"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestAppPosthog_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appposthog secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PosthogApp,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword posthog",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appposthog secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PosthogApp,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AppPosthog.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AppPosthog.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/postman/postman_integration_test.go b/pkg/detectors/postman/postman_integration_test.go
new file mode 100644
index 000000000..65ac16c97
--- /dev/null
+++ b/pkg/detectors/postman/postman_integration_test.go
@@ -0,0 +1,158 @@
+//go:build detectors
+// +build detectors
+
+package postman
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostman_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POSTMAN_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("POSTMAN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postman,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postman,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postman,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postman,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Postman.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Errorf("Postman.FromData() error = %v, wantErr %v", got[i].VerificationError(), tt.wantVerificationErr)
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Postman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/postman/postman_test.go b/pkg/detectors/postman/postman_test.go
index 65ac16c97..10b582888 100644
--- a/pkg/detectors/postman/postman_test.go
+++ b/pkg/detectors/postman/postman_test.go
@@ -1,157 +1,80 @@
-//go:build detectors
-// +build detectors
-
package postman
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPostman_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POSTMAN_TOKEN")
- inactiveSecret := testSecrets.MustGetField("POSTMAN_INACTIVE")
+var (
+ validPattern = "PMAK-aMUVdak2ohpOeeJQVlb1G9EWB09YVQfbvH6YH8HpPRkXHPBg5qYvp9YGOhI"
+ invalidPattern = "PMAK-aMUVdak2ohpOeeJQVlb1G9EWB09?VQfbvH6YH8HpPRkXHPBg5qYvp9YGOhI"
+ keyword = "postman"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPostman_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postman,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postman,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postman secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postman,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postman,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword postman",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Postman.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Errorf("Postman.FromData() error = %v, wantErr %v", got[i].VerificationError(), tt.wantVerificationErr)
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Postman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/postmark/postmark_integration_test.go b/pkg/detectors/postmark/postmark_integration_test.go
new file mode 100644
index 000000000..a9c984c7c
--- /dev/null
+++ b/pkg/detectors/postmark/postmark_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package postmark
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostmark_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POSTMARK_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("POSTMARK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postmark secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postmark,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a postmark secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Postmark,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Postmark.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Postmark.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/postmark/postmark_test.go b/pkg/detectors/postmark/postmark_test.go
index a9c984c7c..8c0e4b6b7 100644
--- a/pkg/detectors/postmark/postmark_test.go
+++ b/pkg/detectors/postmark/postmark_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package postmark
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPostmark_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POSTMARK_TOKEN")
- inactiveSecret := testSecrets.MustGetField("POSTMARK_INACTIVE")
+var (
+ validPattern = "uigi9zkt-7q3u-v1v9-5x2s-91p78wlmsuxv"
+ invalidPattern = "uigi9zkt?7q3u-v1v9-5x2s-91p78wlmsuxv"
+ keyword = "postmark"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPostmark_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postmark secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postmark,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword postmark",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a postmark secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Postmark,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Postmark.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Postmark.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/powrbot/powrbot_integration_test.go b/pkg/detectors/powrbot/powrbot_integration_test.go
new file mode 100644
index 000000000..0b7b184ab
--- /dev/null
+++ b/pkg/detectors/powrbot/powrbot_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package powrbot
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPowrbot_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("POWRBOT_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("POWRBOT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a powrbot secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Powrbot,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a powrbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Powrbot,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Powrbot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Powrbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/powrbot/powrbot_test.go b/pkg/detectors/powrbot/powrbot_test.go
index 0b7b184ab..117a1efd0 100644
--- a/pkg/detectors/powrbot/powrbot_test.go
+++ b/pkg/detectors/powrbot/powrbot_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package powrbot
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPowrbot_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("POWRBOT_TOKEN")
- inactiveSecret := testSecrets.MustGetField("POWRBOT_INACTIVE")
+var (
+ validPattern = "BuvR1k59n67R96aInATbIUbchmLcpLVN10oyqKuB"
+ invalidPattern = "BuvR1k59n67R96a?nATbIUbchmLcpLVN10oyqKuB"
+ keyword = "powrbot"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPowrbot_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a powrbot secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Powrbot,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword powrbot",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a powrbot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Powrbot,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Powrbot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Powrbot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/prefect/prefect_integration_test.go b/pkg/detectors/prefect/prefect_integration_test.go
new file mode 100644
index 000000000..d748c86de
--- /dev/null
+++ b/pkg/detectors/prefect/prefect_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package prefect
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrefect_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PREFECT")
+ inactiveSecret := testSecrets.MustGetField("PREFECT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a prefect secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Prefect,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a prefect secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Prefect,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Prefect.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Prefect.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/prefect/prefect_test.go b/pkg/detectors/prefect/prefect_test.go
index d748c86de..10aa2e077 100644
--- a/pkg/detectors/prefect/prefect_test.go
+++ b/pkg/detectors/prefect/prefect_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package prefect
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPrefect_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PREFECT")
- inactiveSecret := testSecrets.MustGetField("PREFECT_INACTIVE")
+var (
+ validPattern = "pnu_v7mBGSR1R1WT0He5KS2w83RZgInZO8Qwalvr"
+ invalidPattern = "pnu_v7mBGSR1R1WT0He5?S2w83RZgInZO8Qwalvr"
+ keyword = "prefect"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPrefect_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a prefect secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Prefect,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword prefect",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a prefect secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Prefect,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Prefect.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Prefect.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/privacy/privacy_integration_test.go b/pkg/detectors/privacy/privacy_integration_test.go
new file mode 100644
index 000000000..c17317ea6
--- /dev/null
+++ b/pkg/detectors/privacy/privacy_integration_test.go
@@ -0,0 +1,162 @@
+//go:build detectors
+// +build detectors
+
+package privacy
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrivacy_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PRIVACY")
+ inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Privacy,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Privacy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Privacy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Privacy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/privacy/privacy_test.go b/pkg/detectors/privacy/privacy_test.go
index c17317ea6..540db09ca 100644
--- a/pkg/detectors/privacy/privacy_test.go
+++ b/pkg/detectors/privacy/privacy_test.go
@@ -1,161 +1,90 @@
-//go:build detectors
-// +build detectors
-
package privacy
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPrivacy_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PRIVACY")
- inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
+var (
+ validPattern = "sccqe60k-nsak-h1m1-vzq8-vr7u4gxihiwl"
+ invalidPattern = "sccqe60k?nsak-h1m1-vzq8-vr7u4gxihiwl"
+ keyword = "privacy"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPrivacy_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Privacy,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword privacy",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Privacy,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Privacy,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Privacy,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/privatekey/privatekey_integration_test.go b/pkg/detectors/privatekey/privatekey_integration_test.go
new file mode 100644
index 000000000..44c9cf4f9
--- /dev/null
+++ b/pkg/detectors/privatekey/privatekey_integration_test.go
@@ -0,0 +1,216 @@
+//go:build detectors
+// +build detectors
+
+package privatekey
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPrivatekey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
+ secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
+ secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
+ secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: false,
+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",
+ },
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: false,
+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found TLS private key, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(secretTLS),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: true,
+ Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
+ ExtraData: map[string]string{
+ "certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found GitHub SSH private key, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(secretGitHub),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: true,
+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
+ ExtraData: map[string]string{
+ "github_user": "sirdetectsalot",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found encrypted GitHub SSH private key, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(secretGitHubEncrypted),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PrivateKey,
+ Verified: true,
+ Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
+ ExtraData: map[string]string{
+ "github_user": "sirdetectsalot",
+ "encrypted": "true",
+ "cracked_encryption_passphrase": "true",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{IncludeExpired: true}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+func Test_lookupFingerprint(t *testing.T) {
+ tests := []struct {
+ name string
+ publicKeyFingerprintInHex string
+ wantFingerprints bool
+ wantErr bool
+ includeExpired bool
+ }{
+ {
+ name: "got some",
+ publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",
+ wantFingerprints: true,
+ wantErr: false,
+ includeExpired: true,
+ },
+ {
+ name: "got some",
+ publicKeyFingerprintInHex: "none",
+ wantFingerprints: false,
+ wantErr: false,
+ includeExpired: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotFingerprints, err := lookupFingerprint(context.TODO(), tt.publicKeyFingerprintInHex, tt.includeExpired)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
+ t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/privatekey/privatekey_test.go b/pkg/detectors/privatekey/privatekey_test.go
index 44c9cf4f9..661a8fd96 100644
--- a/pkg/detectors/privatekey/privatekey_test.go
+++ b/pkg/detectors/privatekey/privatekey_test.go
@@ -1,215 +1,97 @@
-//go:build detectors
-// +build detectors
-
package privatekey
import (
"context"
"fmt"
- "os"
- "reflect"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPrivatekey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
- secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
- secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
- secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validPattern = `-----BEGIN RSA PRIVATE KEY-----
+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu
+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm
+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k
+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7
+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy
+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs
+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00
+-----END RSA PRIVATE KEY-----
+`
+ invalidPattern = `-----BEGIN?RSA?PRIVATE?KEY-----
+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu
+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm
+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k
+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7
+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy
+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs
+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00
+-----END RSA PRIVATE KEY-----`
+ keyword = "privatekey"
+)
+func TestPrivatekey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find privatekey secret %s within", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PrivateKey,
- Verified: false,
- Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYw",
- },
- {
- DetectorType: detectorspb.DetectorType_PrivateKey,
- Verified: false,
- Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
- },
- },
- wantErr: false,
- },
- {
- name: "found TLS private key, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(secretTLS),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PrivateKey,
- Verified: true,
- Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
- ExtraData: map[string]string{
- "certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
- },
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword privatekey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found GitHub SSH private key, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(secretGitHub),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PrivateKey,
- Verified: true,
- Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
- ExtraData: map[string]string{
- "github_user": "sirdetectsalot",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "found encrypted GitHub SSH private key, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(secretGitHubEncrypted),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PrivateKey,
- Verified: true,
- Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
- ExtraData: map[string]string{
- "github_user": "sirdetectsalot",
- "encrypted": "true",
- "cracked_encryption_passphrase": "true",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{IncludeExpired: true}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PrivatekeyCI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- if os.Getenv("FORCE_PASS_DIFF") == "true" {
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PrivatekeyCI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- })
- }
-}
-
-func Test_lookupFingerprint(t *testing.T) {
- tests := []struct {
- name string
- publicKeyFingerprintInHex string
- wantFingerprints bool
- wantErr bool
- includeExpired bool
- }{
- {
- name: "got some",
- publicKeyFingerprintInHex: "4c5da06caa1c81df9c8e1abe43bac385de1bda76",
- wantFingerprints: true,
- wantErr: false,
- includeExpired: true,
- },
- {
- name: "got some",
- publicKeyFingerprintInHex: "none",
- wantFingerprints: false,
- wantErr: false,
- includeExpired: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- gotFingerprints, err := lookupFingerprint(context.TODO(), tt.publicKeyFingerprintInHex, tt.includeExpired)
- if (err != nil) != tt.wantErr {
- t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
- return
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
- t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/prodpad/prodpad_integration_test.go b/pkg/detectors/prodpad/prodpad_integration_test.go
new file mode 100644
index 000000000..80fd2a5e7
--- /dev/null
+++ b/pkg/detectors/prodpad/prodpad_integration_test.go
@@ -0,0 +1,117 @@
+package prodpad
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestProdpad_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PRODPAD")
+ inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Prodpad,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Prodpad,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/prodpad/prodpad_test.go b/pkg/detectors/prodpad/prodpad_test.go
index 80fd2a5e7..2e5deb815 100644
--- a/pkg/detectors/prodpad/prodpad_test.go
+++ b/pkg/detectors/prodpad/prodpad_test.go
@@ -4,114 +4,88 @@ import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestProdpad_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PRODPAD")
- inactiveSecret := testSecrets.MustGetField("PRODPAD_INACTIVE")
+var (
+ validPattern = "96d3a2234e30b05b6c0ec74df1943b95ec708d09b0c45779874be404ce004335"
+ invalidPattern = "96d3a2234?30b05b6c0ec74df1943b95ec708d09b0c45779874be404ce004335"
+ keyword = "prodpad"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestProdpad_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a prodpad secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Prodpad,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword prodpad",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a prodpad secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Prodpad,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Prodpad.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Prodpad.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/prospectcrm/prospectcrm_integration_test.go b/pkg/detectors/prospectcrm/prospectcrm_integration_test.go
new file mode 100644
index 000000000..af0f8f0fb
--- /dev/null
+++ b/pkg/detectors/prospectcrm/prospectcrm_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package prospectcrm
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestProspectCRM_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PROSPECTCRM")
+ inactiveSecret := testSecrets.MustGetField("PROSPECTCRM_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProspectCRM,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProspectCRM,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ProspectCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ProspectCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/prospectcrm/prospectcrm_test.go b/pkg/detectors/prospectcrm/prospectcrm_test.go
index af0f8f0fb..86175b5af 100644
--- a/pkg/detectors/prospectcrm/prospectcrm_test.go
+++ b/pkg/detectors/prospectcrm/prospectcrm_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package prospectcrm
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestProspectCRM_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PROSPECTCRM")
- inactiveSecret := testSecrets.MustGetField("PROSPECTCRM_INACTIVE")
+var (
+ validPattern = "yawgnnmzs6hqcpdakzpshc79enh55iz0"
+ invalidPattern = "yawgnnmzs6hqcpda?zpshc79enh55iz0"
+ keyword = "prospectcrm"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestProspectCRM_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ProspectCRM,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword prospectcrm",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a prospectcrm secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ProspectCRM,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ProspectCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ProspectCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/protocolsio/protocolsio_integration_test.go b/pkg/detectors/protocolsio/protocolsio_integration_test.go
new file mode 100644
index 000000000..495d2218f
--- /dev/null
+++ b/pkg/detectors/protocolsio/protocolsio_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package protocolsio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestProtocolsIO_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PROTOCOLSIO")
+ inactiveSecret := testSecrets.MustGetField("PROTOCOLSIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProtocolsIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProtocolsIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ProtocolsIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ProtocolsIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/protocolsio/protocolsio_test.go b/pkg/detectors/protocolsio/protocolsio_test.go
index 495d2218f..66d7ae639 100644
--- a/pkg/detectors/protocolsio/protocolsio_test.go
+++ b/pkg/detectors/protocolsio/protocolsio_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package protocolsio
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestProtocolsIO_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PROTOCOLSIO")
- inactiveSecret := testSecrets.MustGetField("PROTOCOLSIO_INACTIVE")
+var (
+ validPattern = "4aphfkag41i9kow1la6gm7sj9zsj5i8xznwlg5j9eq0e1xl3dzr44b19llppaiut"
+ invalidPattern = "4aphfkag41i9kow1la6gm7sj9?sj5i8xznwlg5j9eq0e1xl3dzr44b19llppaiut"
+ keyword = "protocolsio"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestProtocolsIO_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ProtocolsIO,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword protocolsio",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a protocolsio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ProtocolsIO,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ProtocolsIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ProtocolsIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/proxycrawl/proxycrawl_integration_test.go b/pkg/detectors/proxycrawl/proxycrawl_integration_test.go
new file mode 100644
index 000000000..7eea9e70a
--- /dev/null
+++ b/pkg/detectors/proxycrawl/proxycrawl_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package proxycrawl
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestProxyCrawl_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PROXYCRAWL")
+ inactiveSecret := testSecrets.MustGetField("PROXYCRAWL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProxyCrawl,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ProxyCrawl,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ProxyCrawl.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ProxyCrawl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/proxycrawl/proxycrawl_test.go b/pkg/detectors/proxycrawl/proxycrawl_test.go
index 7eea9e70a..31faaa415 100644
--- a/pkg/detectors/proxycrawl/proxycrawl_test.go
+++ b/pkg/detectors/proxycrawl/proxycrawl_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package proxycrawl
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestProxyCrawl_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PROXYCRAWL")
- inactiveSecret := testSecrets.MustGetField("PROXYCRAWL_INACTIVE")
+var (
+ validPattern = "wjXKCFzOWEPRqhTmmbWtVQ"
+ invalidPattern = "wjXKCFzOWEP?qhTmmbWtVQ"
+ keyword = "proxycrawl"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestProxyCrawl_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ProxyCrawl,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword proxycrawl",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a proxycrawl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ProxyCrawl,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ProxyCrawl.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ProxyCrawl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_integration_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_integration_test.go
new file mode 100644
index 000000000..9caa04ae1
--- /dev/null
+++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_integration_test.go
@@ -0,0 +1,160 @@
+//go:build detectors
+// +build detectors
+
+package pubnubpublishkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPubNubPublishKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ pub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_TOKEN")
+ inactivePub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_INACTIVE")
+ sub := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s but not valid", inactivePub, sub)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, would be verified but for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, valid but unexpected api response",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PubNubPublishKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, opts); diff != "" {
+ t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go
index 9caa04ae1..279b26df6 100644
--- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go
+++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go
@@ -1,159 +1,82 @@
-//go:build detectors
-// +build detectors
-
package pubnubpublishkey
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPubNubPublishKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- pub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_TOKEN")
- inactivePub := testSecrets.MustGetField("PUBNUBPUBLISHKEY_INACTIVE")
- sub := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")
+var (
+ validPub = "pub-c-3886bd4t-dwor-vuqm-ond2-4wz4supiwwa2"
+ invalidPub = "p?b-c-3886bd4t-dwor-vuqm-ond2-4wz4supiwwa2"
+ validSub = "sub-c-adky624g-bux6-lz1j-8yrx-8fvodaul9hu5"
+ invalidSub = "sub-c-adky624g-bux6-l?1j-8yrx-8fvodaul9hu5"
+ keyword = "pubnubpublishkey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPubNubPublishKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PubNubPublishKey,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s but not valid", inactivePub, sub)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PubNubPublishKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "found, would be verified but for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PubNubPublishKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - with keyword pubnubpublishkey",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validPub, keyword, validSub),
+ want: []string{validPub + "/" + validSub},
},
{
- name: "found, valid but unexpected api response",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PubNubPublishKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidPub, keyword, invalidSub),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, opts); diff != "" {
- t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_integration_test.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_integration_test.go
new file mode 100644
index 000000000..bbeac7afb
--- /dev/null
+++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pubnubsubscriptionkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPubNubSubscriptionKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PubNubSubscriptionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PubNubSubscriptionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go
index bbeac7afb..54b6b93ac 100644
--- a/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go
+++ b/pkg/detectors/pubnubsubscriptionkey/pubnubsubscriptionkey_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package pubnubsubscriptionkey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPubNubSubscriptionKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("PUBNUBSUBSCRIPTIONKEY_INACTIVE")
+var (
+ validPattern = "sub-c-sy4te40h-w1oc-zpyq-zlrw-w6ehv0y0y78c"
+ invalidPattern = "sub-c-sy4te40h-w1oc-z?yq-zlrw-w6ehv0y0y78c"
+ keyword = "pubnubsubscriptionkey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPubNubSubscriptionKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pubnubsubscriptionkey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pubnubsubscriptionkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PubNubSubscriptionKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PubNubSubscriptionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PubNubSubscriptionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pulumi/pulumi_integration_test.go b/pkg/detectors/pulumi/pulumi_integration_test.go
new file mode 100644
index 000000000..2276bddcf
--- /dev/null
+++ b/pkg/detectors/pulumi/pulumi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pulumi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPulumi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PULUMI")
+ inactiveSecret := testSecrets.MustGetField("PULUMI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pulumi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pulumi,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pulumi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Pulumi,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pulumi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Pulumi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pulumi/pulumi_test.go b/pkg/detectors/pulumi/pulumi_test.go
index 2276bddcf..9783781f1 100644
--- a/pkg/detectors/pulumi/pulumi_test.go
+++ b/pkg/detectors/pulumi/pulumi_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package pulumi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPulumi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PULUMI")
- inactiveSecret := testSecrets.MustGetField("PULUMI_INACTIVE")
+var (
+ validPattern = "pul-xm8r0t347qgoqhrp4c5pq0jkkabeue0x7u35l2ac"
+ invalidPattern = "pul-xm8r0t347qgoqhrp4c?pq0jkkabeue0x7u35l2ac"
+ keyword = "pulumi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPulumi_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pulumi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pulumi,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pulumi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pulumi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Pulumi,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pulumi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Pulumi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/purestake/purestake_integration_test.go b/pkg/detectors/purestake/purestake_integration_test.go
new file mode 100644
index 000000000..90c408e42
--- /dev/null
+++ b/pkg/detectors/purestake/purestake_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package purestake
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPureStake_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PURESTAKE")
+ inactiveSecret := testSecrets.MustGetField("PURESTAKE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a purestake secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PureStake,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a purestake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PureStake,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PureStake.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PureStake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/purestake/purestake_test.go b/pkg/detectors/purestake/purestake_test.go
index 90c408e42..8d2e72cb7 100644
--- a/pkg/detectors/purestake/purestake_test.go
+++ b/pkg/detectors/purestake/purestake_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package purestake
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPureStake_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PURESTAKE")
- inactiveSecret := testSecrets.MustGetField("PURESTAKE_INACTIVE")
+var (
+ validPattern = "uR05tbMNjbrrHCVDGinuNn54QqYiSiSZn5BN3AGW"
+ invalidPattern = "uR05tbM?jbrrHCVDGinuNn54QqYiSiSZn5BN3AGW"
+ keyword = "purestake"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPureStake_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a purestake secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PureStake,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword purestake",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a purestake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PureStake,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PureStake.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PureStake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey_integration_test.go b/pkg/detectors/pushbulletapikey/pushbulletapikey_integration_test.go
new file mode 100644
index 000000000..81aeb603d
--- /dev/null
+++ b/pkg/detectors/pushbulletapikey/pushbulletapikey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package pushbulletapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPushBulletApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PUSHBULLETAPIKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("PUSHBULLETAPIKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PushBulletApiKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PushBulletApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PushBulletApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PushBulletApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go
index 81aeb603d..645ccd65f 100644
--- a/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go
+++ b/pkg/detectors/pushbulletapikey/pushbulletapikey_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package pushbulletapikey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPushBulletApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PUSHBULLETAPIKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("PUSHBULLETAPIKEY_INACTIVE")
+var (
+ validPattern = "S2QPKsMCE3hpR1ac2HYl5EvzdvbrugJCpP"
+ invalidPattern = "S2QPKsMCE3hpR1ac2?Yl5EvzdvbrugJCpP"
+ keyword = "pushbulletapikey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPushBulletApiKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PushBulletApiKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword pushbulletapikey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pushbulletapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PushBulletApiKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PushBulletApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PushBulletApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey_integration_test.go b/pkg/detectors/pusherchannelkey/pusherchannelkey_integration_test.go
new file mode 100644
index 000000000..e6eac1483
--- /dev/null
+++ b/pkg/detectors/pusherchannelkey/pusherchannelkey_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package pusherchannelkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPusherChannelKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ appId := testSecrets.MustGetField("PUSHER_APPID")
+ key := testSecrets.MustGetField("PUSHER_KEY")
+ secret := testSecrets.MustGetField("PUSHER_SECRET")
+ inactiveSecret := testSecrets.MustGetField("PUSHER_SECRET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s within pusherKey %s", secret, appId, key)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PusherChannelKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s but not pusherKey %s valid", inactiveSecret, appId, key)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PusherChannelKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("PusherChannelKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("PusherChannelKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go
index e6eac1483..cd0e9e01f 100644
--- a/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go
+++ b/pkg/detectors/pusherchannelkey/pusherchannelkey_test.go
@@ -1,122 +1,87 @@
-//go:build detectors
-// +build detectors
-
package pusherchannelkey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestPusherChannelKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- appId := testSecrets.MustGetField("PUSHER_APPID")
- key := testSecrets.MustGetField("PUSHER_KEY")
- secret := testSecrets.MustGetField("PUSHER_SECRET")
- inactiveSecret := testSecrets.MustGetField("PUSHER_SECRET_INACTIVE")
+var (
+ validAppId = "4870172"
+ invalidAppId = "487?172"
+ validKey = "8mcc284l3i4sph5a3mgf"
+ invalidKey = "8mcc28?l3i4sph5a3mgf"
+ validSecret = "pg1podgq3qabvh4yd1np"
+ invalidSecret = "pg1podgq3q?bvh4yd1np"
+ validInput = `pusher - '%s'
+ key - '%s'
+ pusher - '%s'`
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestPusherChannelKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s within pusherKey %s", secret, appId, key)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PusherChannelKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf(validInput, validAppId, validKey, validSecret),
+ want: []string{validAppId + validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pusherSecret secret %s within pusherId %s but not pusherKey %s valid", inactiveSecret, appId, key)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PusherChannelKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf(validInput, invalidAppId, invalidKey, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("PusherChannelKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("PusherChannelKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/pypi/pypi_integration_test.go b/pkg/detectors/pypi/pypi_integration_test.go
new file mode 100644
index 000000000..dddf57e41
--- /dev/null
+++ b/pkg/detectors/pypi/pypi_integration_test.go
@@ -0,0 +1,161 @@
+//go:build detectors
+// +build detectors
+
+package pypi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPypi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("PYPI")
+ inactiveSecret := testSecrets.MustGetField("PYPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PyPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PyPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PyPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_PyPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Pypi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Pypi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/pypi/pypi_test.go b/pkg/detectors/pypi/pypi_test.go
index 614da0756..d5c755c0f 100644
--- a/pkg/detectors/pypi/pypi_test.go
+++ b/pkg/detectors/pypi/pypi_test.go
@@ -1,21 +1,13 @@
-//go:build detectors
-// +build detectors
-
package pypi
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPypi_Pattern(t *testing.T) {
@@ -75,146 +67,3 @@ func TestPypi_Pattern(t *testing.T) {
})
}
}
-
-func TestPypi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("PYPI")
- inactiveSecret := testSecrets.MustGetField("PYPI_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PyPI,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pypi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PyPI,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PyPI,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a pypi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_PyPI,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Pypi.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
- }
- }
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Pypi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
- }
- })
- }
-}
diff --git a/pkg/detectors/qase/qase_integration_test.go b/pkg/detectors/qase/qase_integration_test.go
new file mode 100644
index 000000000..271d7d2f9
--- /dev/null
+++ b/pkg/detectors/qase/qase_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package qase
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestQase_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("QASE")
+ inactiveSecret := testSecrets.MustGetField("QASE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a qase secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Qase,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a qase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Qase,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Qase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Qase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/qase/qase_test.go b/pkg/detectors/qase/qase_test.go
index 271d7d2f9..e2f89299c 100644
--- a/pkg/detectors/qase/qase_test.go
+++ b/pkg/detectors/qase/qase_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package qase
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestQase_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("QASE")
- inactiveSecret := testSecrets.MustGetField("QASE_INACTIVE")
+var (
+ validPattern = "X8FDrZHoBELPU4UpeeSIMyQPEi7JyQFsF6oQZi6X"
+ invalidPattern = "X8FDrZHoBELPU4UpeeSI?yQPEi7JyQFsF6oQZi6X"
+ keyword = "qase"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestQase_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a qase secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Qase,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword qase",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a qase secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Qase,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Qase.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Qase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/qualaroo/qualaroo_integration_test.go b/pkg/detectors/qualaroo/qualaroo_integration_test.go
new file mode 100644
index 000000000..92637db43
--- /dev/null
+++ b/pkg/detectors/qualaroo/qualaroo_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package qualaroo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestQualaroo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("QUALAROO_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("QUALAROO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Qualaroo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Qualaroo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Qualaroo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Qualaroo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/qualaroo/qualaroo_test.go b/pkg/detectors/qualaroo/qualaroo_test.go
index 92637db43..0942f1b14 100644
--- a/pkg/detectors/qualaroo/qualaroo_test.go
+++ b/pkg/detectors/qualaroo/qualaroo_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package qualaroo
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestQualaroo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("QUALAROO_TOKEN")
- inactiveSecret := testSecrets.MustGetField("QUALAROO_INACTIVE")
+var (
+ validPattern = "w82SrLORVOsr4VMrPhDG4fih0rYGw86vxGBZch99TVvTfiSBSlpWF6f42BVtf0jI"
+ invalidPattern = "w82SrLORVOsr4VMrPhDG4fih0rYGw86v?GBZch99TVvTfiSBSlpWF6f42BVtf0jI"
+ keyword = "qualaroo"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestQualaroo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Qualaroo,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword qualaroo",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a qualaroo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Qualaroo,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Qualaroo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Qualaroo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/qubole/qubole_integration_test.go b/pkg/detectors/qubole/qubole_integration_test.go
new file mode 100644
index 000000000..ccd8c83e0
--- /dev/null
+++ b/pkg/detectors/qubole/qubole_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package qubole
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestQubole_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("QUBOLE")
+ inactiveSecret := testSecrets.MustGetField("QUBOLE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a qubole secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Qubole,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a qubole secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Qubole,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Qubole.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Qubole.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/qubole/qubole_test.go b/pkg/detectors/qubole/qubole_test.go
index ccd8c83e0..804b49c34 100644
--- a/pkg/detectors/qubole/qubole_test.go
+++ b/pkg/detectors/qubole/qubole_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package qubole
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestQubole_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("QUBOLE")
- inactiveSecret := testSecrets.MustGetField("QUBOLE_INACTIVE")
+var (
+ validPattern = "8p9tm1c0ejc70n8vpef7nq7agfzq7kks73lwxpo29z25ggxszg22ecl9toewbdob"
+ invalidPattern = "8p9tm1c0ejc?0n8vpef7nq7agfzq7kks73lwxpo29z25ggxszg22ecl9toewbdob"
+ keyword = "qubole"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestQubole_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a qubole secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Qubole,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword qubole",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a qubole secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Qubole,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Qubole.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Qubole.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}