Skip to content

Find, verify, and analyze leaked credentials

Added pattern unit tests for detectors starting with the letters r through s (#3752)

Nabeel Alame6be459eb411
diff --git a/pkg/detectors/rabbitmq/rabbitmq_integration_test.go b/pkg/detectors/rabbitmq/rabbitmq_integration_test.go
new file mode 100644
index 000000000..f71148ff1
--- /dev/null
+++ b/pkg/detectors/rabbitmq/rabbitmq_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package rabbitmq
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRabbitMQ_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RABBITMQ")
+ inactiveSecret := testSecrets.MustGetField("RABBITMQ_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RabbitMQ,
+ Verified: true,
+ Redacted: "amqp://rabbit123:*******************@127.0.0.1",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RabbitMQ,
+ Verified: false,
+ Redacted: "amqp://rabbituser:*******************@127.0.0.1",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RabbitMQ.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RabbitMQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rabbitmq/rabbitmq_test.go b/pkg/detectors/rabbitmq/rabbitmq_test.go
index 77a5ffcd6..d21856fd9 100644
--- a/pkg/detectors/rabbitmq/rabbitmq_test.go
+++ b/pkg/detectors/rabbitmq/rabbitmq_test.go
@@ -1,122 +1,86 @@
-//go:build detectors
-// +build detectors
-
package rabbitmq
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRabbitMQ_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RABBITMQ")
- inactiveSecret := testSecrets.MustGetField("RABBITMQ_INACTIVE")
+var (
+ validPattern = "amqps://user123:[email protected]:5672/vhost789"
+ invalidPattern = "amqps://user123:pass456@?abbitmq-host.pattern.com:5672/vhost789"
+ keyword = "rabbitmq"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRabbitMQ_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RabbitMQ,
- Verified: true,
- Redacted: "amqp://rabbit123:*******************@127.0.0.1",
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rabbitmq",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rabbitmq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RabbitMQ,
- Verified: false,
- Redacted: "amqp://rabbituser:*******************@127.0.0.1",
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RabbitMQ.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RabbitMQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/ramp/ramp_integration_test.go b/pkg/detectors/ramp/ramp_integration_test.go
new file mode 100644
index 000000000..375ff3c51
--- /dev/null
+++ b/pkg/detectors/ramp/ramp_integration_test.go
@@ -0,0 +1,163 @@
+//go:build detectors
+// +build detectors
+
+package ramp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRamp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+
+ id := testSecrets.MustGetField("RAMP")
+ secret := testSecrets.MustGetField("RAMP_SECRET")
+ inactiveId := testSecrets.MustGetField("RAMP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ramp client id %s and a ramp secret %s within", id, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ramp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within but not valid", inactiveId, secret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ramp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within", id, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ramp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ramp id %s and secret %s within", id, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ramp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Ramp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Ramp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/ramp/ramp_test.go b/pkg/detectors/ramp/ramp_test.go
index 7989a7509..8969666fc 100644
--- a/pkg/detectors/ramp/ramp_test.go
+++ b/pkg/detectors/ramp/ramp_test.go
@@ -1,163 +1,82 @@
-//go:build detectors
-// +build detectors
-
package ramp
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRamp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
-
- id := testSecrets.MustGetField("RAMP")
- secret := testSecrets.MustGetField("RAMP_SECRET")
- inactiveId := testSecrets.MustGetField("RAMP_INACTIVE")
+var (
+ validKey = "ramp_id_tz8O3QzUBiPcAejdGSIvfcvOxbk1wVfnUEIsieV3"
+ invalidKey = "ramp_id_tz8O3QzUBiPcAejd?SIvfcvOxbk1wVfnUEIsieV3"
+ validSecret = "ramp_sec_maFm2aZYzbZOVNbEy4RiuvREnhZ2Xe19E45hl4A3en20WJyX"
+ invalidSecret = "ramp_sec_maFm2aZYzbZOVNbEy4R?uvREnhZ2Xe19E45hl4A3en20WJyX"
+ keyword = "ramp"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRamp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ramp client id %s and a ramp secret %s within", id, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ramp,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within but not valid", inactiveId, secret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ramp,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ramp id %s and ramp secret %s within", id, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ramp,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - with keyword ramp",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
+ want: []string{validKey + ":" + validSecret},
},
{
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ramp id %s and secret %s within", id, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ramp,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Ramp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Ramp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/rapidapi/rapidapi_integration_test.go b/pkg/detectors/rapidapi/rapidapi_integration_test.go
new file mode 100644
index 000000000..671894841
--- /dev/null
+++ b/pkg/detectors/rapidapi/rapidapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rapidapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRapidApi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RAPIDAPI_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("RAPIDAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RapidApi,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RapidApi,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RapidApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RapidApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rapidapi/rapidapi_test.go b/pkg/detectors/rapidapi/rapidapi_test.go
index 3deee3a8a..956821d36 100644
--- a/pkg/detectors/rapidapi/rapidapi_test.go
+++ b/pkg/detectors/rapidapi/rapidapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package rapidapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRapidApi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RAPIDAPI_TOKEN")
- inactiveSecret := testSecrets.MustGetField("RAPIDAPI_INACTIVE")
+var (
+ validPattern = "HG_Wv9C_fvjMHOUokbSvUCEw3scqZB1Q4LBgw5u8Nb-Khx8HBi"
+ invalidPattern = "HG_Wv9C_fvjMHOUokbSvUCEw3?cqZB1Q4LBgw5u8Nb-Khx8HBi"
+ keyword = "rapidapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRapidApi_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RapidApi,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rapidapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rapidapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RapidApi,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RapidApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RapidApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/raven/raven_integration_test.go b/pkg/detectors/raven/raven_integration_test.go
new file mode 100644
index 000000000..ee8e9a980
--- /dev/null
+++ b/pkg/detectors/raven/raven_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package raven
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRaven_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RAVEN")
+ inactiveSecret := testSecrets.MustGetField("RAVEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a raven secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Raven,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a raven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Raven,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Raven.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Raven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/raven/raven_test.go b/pkg/detectors/raven/raven_test.go
index 0558e15a2..a2089db5a 100644
--- a/pkg/detectors/raven/raven_test.go
+++ b/pkg/detectors/raven/raven_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package raven
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRaven_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RAVEN")
- inactiveSecret := testSecrets.MustGetField("RAVEN_INACTIVE")
+var (
+ validPattern = "JJ4ZIC9EJNWVIS86"
+ invalidPattern = "?J4ZIC9EJNWVIS86"
+ keyword = "raven"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRaven_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a raven secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Raven,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword raven",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a raven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Raven,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Raven.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Raven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rawg/rawg_integration_test.go b/pkg/detectors/rawg/rawg_integration_test.go
new file mode 100644
index 000000000..93b4b7752
--- /dev/null
+++ b/pkg/detectors/rawg/rawg_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rawg
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRawg_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RAWG")
+ inactiveSecret := testSecrets.MustGetField("RAWG_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rawg secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rawg,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rawg secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rawg,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Rawg.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Rawg.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rawg/rawg_test.go b/pkg/detectors/rawg/rawg_test.go
index 8fc636833..986c50eee 100644
--- a/pkg/detectors/rawg/rawg_test.go
+++ b/pkg/detectors/rawg/rawg_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package rawg
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRawg_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RAWG")
- inactiveSecret := testSecrets.MustGetField("RAWG_INACTIVE")
+var (
+ validPattern = "43zgwcor66n9phsxAxxfi7j81fpaecwy"
+ invalidPattern = "43zgwcor66n9phsx?xxfi7j81fpaecwy"
+ keyword = "rawg"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRawg_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rawg secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rawg,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rawg",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rawg secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rawg,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Rawg.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Rawg.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/razorpay/razorpay_integration_test.go b/pkg/detectors/razorpay/razorpay_integration_test.go
new file mode 100644
index 000000000..21703bfff
--- /dev/null
+++ b/pkg/detectors/razorpay/razorpay_integration_test.go
@@ -0,0 +1,109 @@
+//go:build detectors
+// +build detectors
+
+package razorpay
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRazorPay_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secretInactive := testSecrets.MustGetField("RAZORPAY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_live_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RazorPay,
+ Redacted: "rzp_live_SnDTaP1ncfliDt",
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RazorPay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ got[i].RawV2 = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/razorpay/razorpay_test.go b/pkg/detectors/razorpay/razorpay_test.go
index 2e39ec160..b7c6a7897 100644
--- a/pkg/detectors/razorpay/razorpay_test.go
+++ b/pkg/detectors/razorpay/razorpay_test.go
@@ -1,107 +1,82 @@
-//go:build detectors
-// +build detectors
-
package razorpay
import (
"context"
"fmt"
- "os"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRazorPay_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secretInactive := testSecrets.MustGetField("RAZORPAY_INACTIVE")
+var (
+ validKey = "rzp_live_oE9IEkoNYnmwkG"
+ invalidKey = "rzp_live_oE?IEkoNYnmwkG"
+ validSecret = "u5FJFQ0KuTe4hD8iwza076e2"
+ invalidSecret = "u5FJFQ0KuTe4?D8iwza076e2"
+ keyword = "razorpay"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRazorPay_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("25:RAZORPAY_KEY='rzp_live_SnDTaP1ncfliDt'\n\"rzp_secret\" : \" %s\", ", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RazorPay,
- Redacted: "rzp_live_SnDTaP1ncfliDt",
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword razorpay",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
+ want: []string{validKey + validSecret},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RazorPay.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- if os.Getenv("FORCE_PASS_DIFF") == "true" {
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
- got[i].Raw = nil
- got[i].RawV2 = nil
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RazorPay.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/reachmail/reachmail_integration_test.go b/pkg/detectors/reachmail/reachmail_integration_test.go
new file mode 100644
index 000000000..215a22404
--- /dev/null
+++ b/pkg/detectors/reachmail/reachmail_integration_test.go
@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package reachmail
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestReachmail_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REACHMAIL")
+ inactiveSecret := testSecrets.MustGetField("REACHMAIL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a reachmail secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Reachmail,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a reachmail secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Reachmail,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Reachmail.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Reachmail.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/reachmail/reachmail_test.go b/pkg/detectors/reachmail/reachmail_test.go
index f33fa67e1..e18fd5561 100644
--- a/pkg/detectors/reachmail/reachmail_test.go
+++ b/pkg/detectors/reachmail/reachmail_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package reachmail
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestReachmail_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REACHMAIL")
- inactiveSecret := testSecrets.MustGetField("REACHMAIL_INACTIVE")
+var (
+ validPattern = "qFcJR3KhMRWcm1y2O7C1_8pr-VQJgQ8vBvVKmctf8JRr5CFfU6VF9Np2ZMZatF7k"
+ invalidPattern = "qFcJR3KhMRWcm1y2O7C1_8pr-VQJgQ8v?vVKmctf8JRr5CFfU6VF9Np2ZMZatF7k"
+ keyword = "reachmail"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestReachmail_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a reachmail secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Reachmail,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword reachmail",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a reachmail secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Reachmail,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Reachmail.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Reachmail.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/readme/readme_integration_test.go b/pkg/detectors/readme/readme_integration_test.go
new file mode 100644
index 000000000..1ef3c1fe1
--- /dev/null
+++ b/pkg/detectors/readme/readme_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package readme
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestReadMe_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("README")
+ inactiveSecret := testSecrets.MustGetField("README_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a readme secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ReadMe,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a readme secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ReadMe,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ReadMe.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ReadMe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/readme/readme_test.go b/pkg/detectors/readme/readme_test.go
index 1648b60f1..d44751c60 100644
--- a/pkg/detectors/readme/readme_test.go
+++ b/pkg/detectors/readme/readme_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package readme
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestReadMe_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("README")
- inactiveSecret := testSecrets.MustGetField("README_INACTIVE")
+var (
+ validPattern = "rdme_xybq9sovqyexf5wtc437abdsgvw6dfvagdstx3leyr4t2cam7xk0ukv14on7ghun67curt"
+ invalidPattern = "rdme_xybq9sovqyexf5wtc437abds?vw6dfvagdstx3leyr4t2cam7xk0ukv14on7ghun67curt"
+ keyword = "readme"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestReadMe_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a readme secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ReadMe,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword readme",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a readme secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ReadMe,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ReadMe.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ReadMe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems_integration_test.go b/pkg/detectors/reallysimplesystems/reallysimplesystems_integration_test.go
new file mode 100644
index 000000000..197de11b3
--- /dev/null
+++ b/pkg/detectors/reallysimplesystems/reallysimplesystems_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package reallysimplesystems
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestReallySimpleSystems_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS")
+ inactiveSecret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ReallySimpleSystems,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ReallySimpleSystems,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ReallySimpleSystems.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ReallySimpleSystems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go
index 631bac585..2308b10b0 100644
--- a/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go
+++ b/pkg/detectors/reallysimplesystems/reallysimplesystems_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package reallysimplesystems
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestReallySimpleSystems_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS")
- inactiveSecret := testSecrets.MustGetField("REALLYSIMPLESYSTEMS_INACTIVE")
+var (
+ validPattern = "eyt_8-ytglklqvyjGF6dgbXsCSMRLdryUWQCSQ1n1uSph1Ppl.jMeMx0EkxVLcmbKPTi7kPYKzNa04V96eiJlV.XxtKFDKTJCkcThvKR2x0ctVUL.FrbHVXEx55VZZPD0FKW4onNDO05Ab_9cbnrLTr4d--.eyX.hZw.Q5g-pvyWS5jjDUyiE60WZSgQq4uM8PUVw1-OJegMnGGLEKq48G5X7zgQAcYvUIRQy4Ku.gKQkNFIL4Nlj3ob7Bi4VUDs_DjvtS_Jxho-Hn_hgIhfYQVw19wQCcc8m7v2jCXz2MTi74vpzW7aY2C123i5uFr1KdVQLX.Xlf5ZtuLAsYZ5MsG1G6VYO_t7l6nl8D-Y3fml7uSGPXkeQT7AW1HKOTrBdGj0L_OMMYBaZ4-R9tnUjM7Jf1LExVn2NIQnNGR7Jyl69WigxQGLaUbNvE9DQkVtV4ar0FORi8mrwAQdb5UxZkOxW4me_F189Y-rs0h9q4vrjmM3Fu1Uj_UCrV4IP6AYr4b9yXzgAlcybQSNUISltLwYC.07LKjRy.dLvF-A2b5mIjyKbR3rOUmiXfeAveBRakkEOKSNOSTDVM839O9a-8tqAggj8eDdPunO8Pg6L_Af_eJowZNhtRluVuFDTHuUW.lAG1bqm.ZJQak800cspbDbB8InwOb0V1UvoO.c.ZOAWM3fQVoidNeDbKbWKR3JdE3iWCGsqz89Zdd6LkxwHAd5TRLn.ZVg67d1.g2S3kadXrTZ0qN6o_7apQ38PbGKydnCTLMfdgN6uIPqAtQntsF6YNvNfFHnXgsfhTkVUDJc5g6oQB_jDeGa8rkCmH6_DO6Z1qV04G.fj.QLJIgMhcdGL8Y3NMRv87spZxNWmX1glZ1AYQlWbbqwzAC1v-p0ImXQSuq6G8aIt8e4w7FnOmOn_Q0NwD85JRgdy.W4f.5hX-zLhj1RDqvC1-Ebd2OLDdne0xqMMNe4kGnzQ6dpDtbZXPcim443CyFxaFD_DK5-Iupc032AKV_Sm9eDSlRuAmAQ7LhvypNWi_6e9NuVUEQ5P6HfBe1eXkZ3y5HkZpWbJqJW2D"
+ invalidPattern = "ayt_8-ytglklqvyjGF6dgbXsCSMRLdryUWQCSQ1n1uSph1Ppl.jMeMx0EkxVLcmbKPTi7kPYKzNa04V96eiJlV.XxtKFDKTJCkcThvKR2x0ctVUL.FrbHVXEx55VZZPD0FKW4onNDO05Ab_9cbnrLTr4d--.eyX.hZw.Q5g-pvyWS5jjDUyiE60WZSgQq4uM8PUVw1-OJegMnGGLEKq48G5X7zgQAcYvUIRQy4Ku.gKQkNFIL4Nlj3ob7Bi4VUDs_DjvtS_Jxho-Hn_hgIhfYQVw19wQCcc8m7v2jCXz2MTi74vpzW7aY2C123i5uFr1KdVQLX.Xlf5ZtuLAsYZ5MsG1G6VYO_t7l6nl8D-Y3fml7uSGPXkeQT7AW1HKOTrBdGj0L_OMMYBaZ4-R9tnUjM7Jf1LExVn2NIQnNGR7Jyl69WigxQGLaUbNvE9DQkVtV4ar0FORi8mrwAQdb5UxZkOxW4me_F189Y-rs0h9q4vrjmM3Fu1Uj_UCrV4IP6AYr4b9yXzgAlcybQSNUISltLwYC.07LKjRy.dLvF-A2b5mIjyKbR3rOUmiXfeAveBRakkEOKSNOSTDVM839O9a-8tqAggj8eDdPunO8Pg6L_Af_eJowZNhtRluVuFDTHuUW.lAG1bqm.ZJQak800cspbDbB8InwOb0V1UvoO.c.ZOAWM3fQVoidNeDbKbWKR3JdE3iWCGsqz89Zdd6LkxwHAd5TRLn.ZVg67d1.g2S3kadXrTZ0qN6o_7apQ38PbGKydnCTLMfdgN6uIPqAtQntsF6YNvNfFHnXgsfhTkVUDJc5g6oQB_jDeGa8rkCmH6_DO6Z1qV04G.fj.QLJIgMhcdGL8Y3NMRv87spZxNWmX1glZ1AYQlWbbqwzAC1v-p0ImXQSuq6G8aIt8e4w7FnOmOn_Q0NwD85JRgdy.W4f.5hX-zLhj1RDqvC1-Ebd2OLDdne0xqMMNe4kGnzQ6dpDtbZXPcim443CyFxaFD_DK5-Iupc032AKV_Sm9eDSlRuAmAQ7LhvypNWi_6e9NuVUEQ5P6HfBe1eXkZ3y5HkZpWbJqJW2D"
+ keyword = "reallysimplesystems"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestReallySimpleSystems_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ReallySimpleSystems,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword reallysimplesystems",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a reallysimplesystems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ReallySimpleSystems,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ReallySimpleSystems.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ReallySimpleSystems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rebrandly/rebrandly_integration_test.go b/pkg/detectors/rebrandly/rebrandly_integration_test.go
new file mode 100644
index 000000000..a9abde619
--- /dev/null
+++ b/pkg/detectors/rebrandly/rebrandly_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rebrandly
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRebrandly_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REBRANDLY")
+ inactiveSecret := testSecrets.MustGetField("REBRANDLY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rebrandly,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rebrandly,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Rebrandly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Rebrandly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rebrandly/rebrandly_test.go b/pkg/detectors/rebrandly/rebrandly_test.go
index f1b85b2ad..5ac8e8ce5 100644
--- a/pkg/detectors/rebrandly/rebrandly_test.go
+++ b/pkg/detectors/rebrandly/rebrandly_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package rebrandly
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRebrandly_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REBRANDLY")
- inactiveSecret := testSecrets.MustGetField("REBRANDLY_INACTIVE")
+var (
+ validPattern = "_dd9gKTi7gDhQYowXu5efwPLKkAAQ4qP"
+ invalidPattern = "_dd9gKTi7gDhQYow?u5efwPLKkAAQ4qP"
+ keyword = "rebrandly"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRebrandly_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rebrandly,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rebrandly",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rebrandly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rebrandly,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Rebrandly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Rebrandly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rechargepayments/rechargepayments_integration_test.go b/pkg/detectors/rechargepayments/rechargepayments_integration_test.go
new file mode 100644
index 000000000..57492ecee
--- /dev/null
+++ b/pkg/detectors/rechargepayments/rechargepayments_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rechargepayments
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRechargePayments_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RECHARGEPAYMENTS")
+ inactiveSecret := testSecrets.MustGetField("RECHARGEPAYMENTS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RechargePayments,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RechargePayments,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RechargePayments.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RechargePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rechargepayments/rechargepayments_test.go b/pkg/detectors/rechargepayments/rechargepayments_test.go
index 919b73909..d93028af5 100644
--- a/pkg/detectors/rechargepayments/rechargepayments_test.go
+++ b/pkg/detectors/rechargepayments/rechargepayments_test.go
@@ -1,120 +1,85 @@
-//go:build detectors
-// +build detectors
-
package rechargepayments
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRechargePayments_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RECHARGEPAYMENTS")
- inactiveSecret := testSecrets.MustGetField("RECHARGEPAYMENTS_INACTIVE")
+var (
+ validApiKey = "sk_test_3x3_2fbA8B8f75b3f74D3caEF6ec023c3AAbbe48Edcc2393dc343688Dfa8bbcb141f"
+ invalidApiKey = "sk_test_3x3_?fbA8B8f75b3f74D3caEF6ec023c3AAbbe48Edcc2393dc343688Dfa8bbcb141f"
+ validOldApiSha = "09ECafFABafCeDec29b51B425E1Ed5BC70c6C88c8CeddDe0ba7ab0Af"
+ invalidOldApiSha = "0?ECafFABafCeDec29b51B425E1Ed5BC70c6C88c8CeddDe0ba7ab0Af"
+ validOldApiMd5 = "FCd1AAaD6EF3D4bCFdaB9c831cBc0d57"
+ invalidOldApiMd5 = "F?d1AAaD6EF3D4bCFdaB9c831cBc0d57"
+ keyword = "rechargepayments"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRechargePayments_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RechargePayments,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rechargepayments",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validApiKey, keyword, validOldApiSha, keyword, validOldApiMd5),
+ want: []string{validApiKey, validOldApiSha, validOldApiMd5},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rechargepayments secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RechargePayments,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidApiKey, keyword, invalidOldApiSha, keyword, invalidOldApiMd5),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RechargePayments.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RechargePayments.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/redis/redis_integration_test.go b/pkg/detectors/redis/redis_integration_test.go
index caebf3bb8..85ff1b8ff 100644
--- a/pkg/detectors/redis/redis_integration_test.go
+++ b/pkg/detectors/redis/redis_integration_test.go
@@ -154,3 +154,78 @@ func TestRedisIntegration_FromChunk(t *testing.T) {
})
}
}
+
+func TestURI_FromChunk(t *testing.T) {
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "bad scheme",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("file://user:[email protected]:123/wh/at/ever"),
+ verify: true,
+ },
+ wantErr: false,
+ },
+ {
+ name: "unverified Redis",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("redis://user:[email protected]"),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Redis,
+ Verified: false,
+ Redacted: "redis://user:*******@redis.com",
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.go
index 92b58dece..fccd171ab 100644
--- a/pkg/detectors/redis/redis_test.go
+++ b/pkg/detectors/redis/redis_test.go
@@ -1,88 +1,86 @@
-//go:build detectors
-// +build detectors
-
package redis
import (
"context"
+ "fmt"
"testing"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestURI_FromChunk(t *testing.T) {
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validKey = "redis://1SHPg6BjyS:$YYbMDot25P@PvfdX9VshFjW"
+ invalidKey = "redis://1SHPg6BjySX$YYbMDot25P@PvfdX9VshFjW"
+ validDomain = "QZc67koZ6rUV.redis.cache.windows.net:6380"
+ invalidDomain = "QZc67koZ6rUV.redis.cache.windows.net:6381"
+ password = "Xcc3S9d7And6aMdfOcUc0acHJh3CiDh3l9DsapNwGwyS"
+ validAzureRedis = validDomain + ",password=" + password + ",ssl=True,abortConnect=False"
+ invalidAzureRedis = invalidDomain + ",password=" + password + ",ssl=False,abortConnect=True"
+ keyword = "redis"
+)
+
+func TestRedisIntegration_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "bad scheme",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("file://user:[email protected]:123/wh/at/ever"),
- verify: true,
- },
- wantErr: false,
+ name: "valid pattern - with keyword redis",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validAzureRedis),
+ want: []string{"rediss://:" + password + "@" + validDomain},
},
{
- name: "unverified Redis",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("redis://user:[email protected]"),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Redis,
- Verified: false,
- Redacted: "redis://user:*******@redis.com",
- },
- },
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidAzureRedis),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/refiner/refiner_integration_test.go b/pkg/detectors/refiner/refiner_integration_test.go
new file mode 100644
index 000000000..6bc0969cb
--- /dev/null
+++ b/pkg/detectors/refiner/refiner_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package refiner
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRefiner_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REFINER")
+ inactiveSecret := testSecrets.MustGetField("REFINER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a refiner secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Refiner,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a refiner secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Refiner,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Refiner.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Refiner.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/refiner/refiner_test.go b/pkg/detectors/refiner/refiner_test.go
index 68a9207de..fc1e9b41b 100644
--- a/pkg/detectors/refiner/refiner_test.go
+++ b/pkg/detectors/refiner/refiner_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package refiner
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRefiner_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REFINER")
- inactiveSecret := testSecrets.MustGetField("REFINER_INACTIVE")
+var (
+ validPattern = "2341fb6f-71d4-dbbc-1317-98c28ceb5837"
+ invalidPattern = "2341fb6f?71d4-dbbc-1317-98c28ceb5837"
+ keyword = "refiner"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRefiner_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a refiner secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Refiner,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword refiner",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a refiner secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Refiner,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Refiner.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Refiner.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rentman/rentman_integration_test.go b/pkg/detectors/rentman/rentman_integration_test.go
new file mode 100644
index 000000000..b7a04ede9
--- /dev/null
+++ b/pkg/detectors/rentman/rentman_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rentman
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRentman_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RENTMAN")
+ inactiveSecret := testSecrets.MustGetField("RENTMAN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rentman secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rentman,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rentman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rentman,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Rentman.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Rentman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rentman/rentman_test.go b/pkg/detectors/rentman/rentman_test.go
index 5bd678616..1ae29bdeb 100644
--- a/pkg/detectors/rentman/rentman_test.go
+++ b/pkg/detectors/rentman/rentman_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package rentman
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRentman_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RENTMAN")
- inactiveSecret := testSecrets.MustGetField("RENTMAN_INACTIVE")
+var (
+ validPattern = "eytuZDhcszM3EZO7iPeRfJfZMoRMLXe7qoNV.eyA3uoaQ3ttxNcElupuqjIGN2K-bLTq_RVtB6w6ySxFCOttLniQBf84yvwaBBjw4W1veuikC2uWzoW5Gtw-CpEC-.35zBGkn7t7G19FduNmXTy1r_8bXME-VD5xIflYzioiZvyRLfy685qs8TQDIJV7zLi98cPLrLTE_yXwmppu9o88_OWPzh9oAqmVGqvX5Z_1RXqd6e5MsaTFx22tqKw14njwD.qvyujp.NR6QOQmyREUt2XnUtqdupyGX"
+ invalidPattern = "aytuZDhcszM3EZO7iPeRfJfZMoRMLXe7qoNV.eyA3uoaQ3ttxNcElupuqjIGN2K-bLTq_RVtB6w6ySxFCOttLniQBf84yvwaBBjw4W1veuikC2uWzoW5Gtw-CpEC-.35zBGkn7t7G19FduNmXTy1r_8bXME-VD5xIflYzioiZvyRLfy685qs8TQDIJV7zLi98cPLrLTE_yXwmppu9o88_OWPzh9oAqmVGqvX5Z_1RXqd6e5MsaTFx22tqKw14njwD.qvyujp.NR6QOQmyREUt2XnUtqdupyG?"
+ keyword = "rentman"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRentman_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rentman secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rentman,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rentman",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rentman secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rentman,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Rentman.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Rentman.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/repairshopr/repairshopr_integration_test.go b/pkg/detectors/repairshopr/repairshopr_integration_test.go
new file mode 100644
index 000000000..a6c59bddb
--- /dev/null
+++ b/pkg/detectors/repairshopr/repairshopr_integration_test.go
@@ -0,0 +1,118 @@
+//go:build detectors
+// +build detectors
+
+package repairshopr
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRepairshopr_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REPAIRSHOPR")
+ domain := testSecrets.MustGetField("REPAIRSHOPR_DOMAIN")
+ inactiveSecret := testSecrets.MustGetField("REPAIRSHOPR_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Repairshopr,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Repairshopr,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Repairshopr.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Repairshopr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/repairshopr/repairshopr_test.go b/pkg/detectors/repairshopr/repairshopr_test.go
index a6c59bddb..4d18e65ea 100644
--- a/pkg/detectors/repairshopr/repairshopr_test.go
+++ b/pkg/detectors/repairshopr/repairshopr_test.go
@@ -1,117 +1,82 @@
-//go:build detectors
-// +build detectors
-
package repairshopr
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRepairshopr_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REPAIRSHOPR")
- domain := testSecrets.MustGetField("REPAIRSHOPR_DOMAIN")
- inactiveSecret := testSecrets.MustGetField("REPAIRSHOPR_INACTIVE")
+var (
+ validKey = "sMpDWS14xuMP2E1FMCufXKajIsodvYiuGT9lWll8gYMAYFDRgOD"
+ invalidKey = "?MpDWS14xuMP2E1FMCufXKajIsodvYiuGT9lWll8gYMAYFDRgOD"
+ validDomain = "uI8mmO1KgWwz"
+ invalidDomain = "?I8mmO1KgWw?"
+ keyword = "repairshopr"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRepairshopr_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s", secret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Repairshopr,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword repairshopr",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),
+ want: []string{validKey, validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a repairshopr secret %s within repairshoprdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Repairshopr,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Repairshopr.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Repairshopr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/replicate/replicate_integration_test.go b/pkg/detectors/replicate/replicate_integration_test.go
new file mode 100644
index 000000000..1e9b4c90b
--- /dev/null
+++ b/pkg/detectors/replicate/replicate_integration_test.go
@@ -0,0 +1,127 @@
+//go:build detectors
+// +build detectors
+
+package replicate
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestReplicate_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REPLICATE")
+ inactiveSecret := testSecrets.MustGetField("REPLICATE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a replicate secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Replicate,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a replicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Replicate,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Replicate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Replicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/replicate/replicate_test.go b/pkg/detectors/replicate/replicate_test.go
index ab597813d..d04c0486b 100644
--- a/pkg/detectors/replicate/replicate_test.go
+++ b/pkg/detectors/replicate/replicate_test.go
@@ -1,127 +1,80 @@
-//go:build detectors
-// +build detectors
-
package replicate
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestReplicate_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REPLICATE")
- inactiveSecret := testSecrets.MustGetField("REPLICATE_INACTIVE")
+var (
+ validPattern = "r8_7DTl1j7Nsy6dpc3GGwkd8qsWFl33hy4ICtNzH"
+ invalidPattern = "r8_7DTl1j7Nsy6dpc3GG?kd8qsWFl33hy4ICtNzH"
+ keyword = "replicate"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestReplicate_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a replicate secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Replicate,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword replicate",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a replicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Replicate,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Replicate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Replicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/replyio/replyio_integration_test.go b/pkg/detectors/replyio/replyio_integration_test.go
new file mode 100644
index 000000000..798fd20cf
--- /dev/null
+++ b/pkg/detectors/replyio/replyio_integration_test.go
@@ -0,0 +1,127 @@
+//go:build detectors
+// +build detectors
+
+package replyio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestReplyio_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REPLYIO")
+ inactiveSecret := testSecrets.MustGetField("REPLYIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a replyio secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ReplyIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a replyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ReplyIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Replyio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Replyio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/replyio/replyio_test.go b/pkg/detectors/replyio/replyio_test.go
index 9f01fa023..d1411db91 100644
--- a/pkg/detectors/replyio/replyio_test.go
+++ b/pkg/detectors/replyio/replyio_test.go
@@ -1,127 +1,90 @@
-//go:build detectors
-// +build detectors
-
package replyio
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestReplyio_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REPLYIO")
- inactiveSecret := testSecrets.MustGetField("REPLYIO_INACTIVE")
+var (
+ validPattern = "Y8rVcIMii3FFhyQmzEDfttno"
+ invalidPattern = "Y8rVcIMii3FF?yQmzEDfttno"
+ keyword = "replyio"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestReplyio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a replyio secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ReplyIO,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword replyio",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a replyio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ReplyIO,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Replyio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Replyio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/requestfinance/requestfinance_integration_test.go b/pkg/detectors/requestfinance/requestfinance_integration_test.go
new file mode 100644
index 000000000..f6c79fb8f
--- /dev/null
+++ b/pkg/detectors/requestfinance/requestfinance_integration_test.go
@@ -0,0 +1,127 @@
+//go:build detectors
+// +build detectors
+
+package requestfinance
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRequestfinance_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REQUESTFINANCE")
+ inactiveSecret := testSecrets.MustGetField("REQUESTFINANCE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RequestFinance,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RequestFinance,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Requestfinance.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Requestfinance.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/requestfinance/requestfinance_test.go b/pkg/detectors/requestfinance/requestfinance_test.go
index d8dc273f8..4f0b3e044 100644
--- a/pkg/detectors/requestfinance/requestfinance_test.go
+++ b/pkg/detectors/requestfinance/requestfinance_test.go
@@ -1,127 +1,90 @@
-//go:build detectors
-// +build detectors
-
package requestfinance
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRequestfinance_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REQUESTFINANCE")
- inactiveSecret := testSecrets.MustGetField("REQUESTFINANCE_INACTIVE")
+var (
+ validPattern = "IZAT6SA-AJUQES1-BZWZYWJ-VDIXPVG"
+ invalidPattern = "IZAT6SA?AJUQES1-BZWZYWJ-VDIXPVG"
+ keyword = "requestfinance"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRequestfinance_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RequestFinance,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword requestfinance",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a requestfinance secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RequestFinance,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Requestfinance.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Requestfinance.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/restpack/restpack_integration_test.go b/pkg/detectors/restpack/restpack_integration_test.go
new file mode 100644
index 000000000..05399149d
--- /dev/null
+++ b/pkg/detectors/restpack/restpack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package restpack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRestpack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RESTPACK")
+ inactiveSecret := testSecrets.MustGetField("RESTPACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a restpack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Restpack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a restpack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Restpack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Restpack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Restpack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/restpack/restpack_test.go b/pkg/detectors/restpack/restpack_test.go
index ff3404f3c..e90b8c366 100644
--- a/pkg/detectors/restpack/restpack_test.go
+++ b/pkg/detectors/restpack/restpack_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package restpack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRestpack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RESTPACK")
- inactiveSecret := testSecrets.MustGetField("RESTPACK_INACTIVE")
+var (
+ validPattern = "RGTuhewdnuYUbL6cFbVTD2DUgVG11qDzuzV1c4gBEUFQaelZ"
+ invalidPattern = "RGTuhewdnuYUbL6cFbVTD2DU?VG11qDzuzV1c4gBEUFQaelZ"
+ keyword = "restpack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRestpack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a restpack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Restpack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword restpack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a restpack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Restpack,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Restpack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Restpack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_integration_test.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_integration_test.go
new file mode 100644
index 000000000..db81cb172
--- /dev/null
+++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package restpackhtmltopdfapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRestpackHtmlToPdfAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI")
+ inactiveSecret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RestpackHtmlToPdfAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RestpackHtmlToPdfAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go
index 9666074ea..a8e3c4522 100644
--- a/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go
+++ b/pkg/detectors/restpackhtmltopdfapi/restpackhtmltopdfapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package restpackhtmltopdfapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRestpackHtmlToPdfAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI")
- inactiveSecret := testSecrets.MustGetField("RESTPACKHTMLTOPDFAPI_INACTIVE")
+var (
+ validPattern = "vVIKB7ypMMXOlpRx2ad1g1t8JmvDBsvQ02LMc1nuNOQYAMEF"
+ invalidPattern = "vVIKB7ypMMXOlpRx2ad1g1t8?mvDBsvQ02LMc1nuNOQYAMEF"
+ keyword = "restpackhtmltopdfapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRestpackHtmlToPdfAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword restpackhtmltopdfapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a restpackhtmltopdfapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RestpackHtmlToPdfAPI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RestpackHtmlToPdfAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RestpackHtmlToPdfAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_integration_test.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_integration_test.go
new file mode 100644
index 000000000..36a3816a5
--- /dev/null
+++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package restpackscreenshotapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRestpackScreenshotAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI")
+ inactiveSecret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RestpackScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RestpackScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go
index 36a3816a5..04fda6686 100644
--- a/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go
+++ b/pkg/detectors/restpackscreenshotapi/restpackscreenshotapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package restpackscreenshotapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRestpackScreenshotAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI")
- inactiveSecret := testSecrets.MustGetField("RESTPACKSCREENSHOTAPI_INACTIVE")
+var (
+ validPattern = "O6xSfj5GgiV3imFS5IU13903DwIfCSnawlBybGQ9sMKg9Ugk"
+ invalidPattern = "O6xSfj5GgiV3imFS5IU13903?wIfCSnawlBybGQ9sMKg9Ugk"
+ keyword = "restpackscreenshotapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRestpackScreenshotAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword restpackscreenshotapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a restpackscreenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RestpackScreenshotAPI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RestpackScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RestpackScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rev/rev_integration_test.go b/pkg/detectors/rev/rev_integration_test.go
new file mode 100644
index 000000000..4b9b34ef5
--- /dev/null
+++ b/pkg/detectors/rev/rev_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package rev
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRev_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REV")
+ inactiveSecret := testSecrets.MustGetField("REV_INACTIVE")
+ clientSecret := testSecrets.MustGetField("REV_CLIENT")
+ inactiveClientSecret := testSecrets.MustGetField("REV_CLIENT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within", secret, clientSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rev,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within but not valid", inactiveSecret, inactiveClientSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rev,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Rev.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Rev.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rev/rev_test.go b/pkg/detectors/rev/rev_test.go
index 885a1dace..e7b0bc84f 100644
--- a/pkg/detectors/rev/rev_test.go
+++ b/pkg/detectors/rev/rev_test.go
@@ -1,122 +1,83 @@
-//go:build detectors
-// +build detectors
-
package rev
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRev_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REV")
- inactiveSecret := testSecrets.MustGetField("REV_INACTIVE")
- clientSecret := testSecrets.MustGetField("REV_CLIENT")
- inactiveClientSecret := testSecrets.MustGetField("REV_CLIENT_INACTIVE")
+var (
+ validUserKey = "T5LJHzWwZzdpmNtxKDaDcDUlckU="
+ invalidUserKey = "T5LJHzWwZzdpmN?xKDaDcDUlckU="
+ validClientKey = "utD8J485zVIeRCbHyU8yYfw55m2"
+ invalidClientKey = "utD8J485zVIeRC?HyU8yYfw55m2"
+ keyword = "rev"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRev_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within", secret, clientSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rev,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rev",
+ input: fmt.Sprintf("%s token - '%s '\n%s token - '%s '\n", keyword, validUserKey, keyword, validClientKey),
+ want: []string{validUserKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rev secret %s with rev client secret %s within but not valid", inactiveSecret, inactiveClientSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rev,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s '\n%s token - '%s '\n", keyword, invalidUserKey, keyword, invalidClientKey),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Rev.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Rev.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/revampcrm/revampcrm_integration_test.go b/pkg/detectors/revampcrm/revampcrm_integration_test.go
new file mode 100644
index 000000000..84ab2dde4
--- /dev/null
+++ b/pkg/detectors/revampcrm/revampcrm_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package revampcrm
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRevampCRM_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("REVAMPCRM")
+ user := testSecrets.MustGetField("ACCOUNT_USER")
+ inactiveSecret := testSecrets.MustGetField("REVAMPCRM_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s", secret, user)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RevampCRM,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RevampCRM,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RevampCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RevampCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/revampcrm/revampcrm_test.go b/pkg/detectors/revampcrm/revampcrm_test.go
index 1101321f6..e165ecc91 100644
--- a/pkg/detectors/revampcrm/revampcrm_test.go
+++ b/pkg/detectors/revampcrm/revampcrm_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package revampcrm
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRevampCRM_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("REVAMPCRM")
- user := testSecrets.MustGetField("ACCOUNT_USER")
- inactiveSecret := testSecrets.MustGetField("REVAMPCRM_INACTIVE")
+var (
+ validKey = "JFjlJg7KqFGp5RmsJVq7WDmwObgHW6JEtMOitGuP"
+ invalidKey = "JFjlJg7KqFGp5RmsJVq7?DmwObgHW6JEtMOitGuP"
+ validId = "JzNRi@oQyzxG>LG7f2oe7R.Xf"
+ invalidId = "JzNRi@oQyzxG!LG7f2oe7R.Xf"
+ keyword = "revampcrm"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRevampCRM_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s", secret, user)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RevampCRM,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword revampcrm",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a revampcrm secret %s within revampcrm %s but not valid", inactiveSecret, user)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RevampCRM,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RevampCRM.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RevampCRM.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/ringcentral/ringcentral_integration_test.go b/pkg/detectors/ringcentral/ringcentral_integration_test.go
new file mode 100644
index 000000000..e27386f2c
--- /dev/null
+++ b/pkg/detectors/ringcentral/ringcentral_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package ringcentral
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRingcentral_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RINGCENTRAL")
+ uri := testSecrets.MustGetField("REDIRECTURI")
+ inactiveSecret := testSecrets.MustGetField("RINGCENTRAL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s", secret, uri)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RingCentral,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s but not valid", inactiveSecret, uri)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RingCentral,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Ringcentral.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Ringcentral.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/ringcentral/ringcentral_test.go b/pkg/detectors/ringcentral/ringcentral_test.go
index 360028638..5939da8fc 100644
--- a/pkg/detectors/ringcentral/ringcentral_test.go
+++ b/pkg/detectors/ringcentral/ringcentral_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package ringcentral
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRingcentral_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RINGCENTRAL")
- uri := testSecrets.MustGetField("REDIRECTURI")
- inactiveSecret := testSecrets.MustGetField("RINGCENTRAL_INACTIVE")
+var (
+ validKey = "OTXwnia5kOs-SlF3kZEFJK"
+ invalidKey = "OTXwnia5kOs?SlF3kZEFJK"
+ validUri = "https://wwwSG7$com"
+ invalidUri = "https://?wwSG7$com"
+ keyword = "ringcentral"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRingcentral_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s", secret, uri)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RingCentral,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword ringcentral",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validUri),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ringcentral secret %s within ringcentral uri %s but not valid", inactiveSecret, uri)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RingCentral,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidUri),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Ringcentral.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Ringcentral.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/ritekit/ritekit_integration_test.go b/pkg/detectors/ritekit/ritekit_integration_test.go
new file mode 100644
index 000000000..56bfcfd35
--- /dev/null
+++ b/pkg/detectors/ritekit/ritekit_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package ritekit
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRiteKit_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RITEKIT")
+ inactiveSecret := testSecrets.MustGetField("RITEKIT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ritekit secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RiteKit,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ritekit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RiteKit,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RiteKit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RiteKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/ritekit/ritekit_test.go b/pkg/detectors/ritekit/ritekit_test.go
index 063651feb..438ac5ed9 100644
--- a/pkg/detectors/ritekit/ritekit_test.go
+++ b/pkg/detectors/ritekit/ritekit_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package ritekit
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRiteKit_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RITEKIT")
- inactiveSecret := testSecrets.MustGetField("RITEKIT_INACTIVE")
+var (
+ validPattern = "6c2428ff83181c3dcf94e97cb0b15b23599a3cb22857"
+ invalidPattern = "6c2428ff83181c3dcf94e9?cb0b15b23599a3cb22857"
+ keyword = "ritekit"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRiteKit_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ritekit secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RiteKit,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword ritekit",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ritekit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RiteKit,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RiteKit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RiteKit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/roaring/roaring_integration_test.go b/pkg/detectors/roaring/roaring_integration_test.go
new file mode 100644
index 000000000..2be07072f
--- /dev/null
+++ b/pkg/detectors/roaring/roaring_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package roaring
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRoaring_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ client := testSecrets.MustGetField("ROARING_CLIENT")
+ secret := testSecrets.MustGetField("ROARING_SECRET")
+ inactiveClient := testSecrets.MustGetField("ROARING_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a roaring secret %s within roaring client %s", secret, client)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Roaring,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a roaring client %s within roaring secret %s but not valid", inactiveClient, secret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Roaring,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Roaring.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Roaring.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/roaring/roaring_test.go b/pkg/detectors/roaring/roaring_test.go
index 640ddb524..fec47e026 100644
--- a/pkg/detectors/roaring/roaring_test.go
+++ b/pkg/detectors/roaring/roaring_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package roaring
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRoaring_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- client := testSecrets.MustGetField("ROARING_CLIENT")
- secret := testSecrets.MustGetField("ROARING_SECRET")
- inactiveClient := testSecrets.MustGetField("ROARING_INACTIVE")
+var (
+ validClient = "-3o_mYkUEBbUKtKjGCg5tgnkzU4T"
+ invalidClient = "-3o_mYkUEBbU?tKjGCg5tgnkzU4T"
+ validSecret = "XqTQqnAkyH6We_TEK1CRjhwRV51h"
+ invalidSecret = "Xq?QqnAkyH6We_TEK1CRjhwRV51h"
+ keyword = "roaring"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRoaring_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a roaring secret %s within roaring client %s", secret, client)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Roaring,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword roaring",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validClient, keyword, validSecret),
+ want: []string{validSecret + validSecret},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a roaring client %s within roaring secret %s but not valid", inactiveClient, secret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Roaring,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidClient, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Roaring.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Roaring.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rocketreach/rocketreach_integration_test.go b/pkg/detectors/rocketreach/rocketreach_integration_test.go
new file mode 100644
index 000000000..651f1c2a8
--- /dev/null
+++ b/pkg/detectors/rocketreach/rocketreach_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rocketreach
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRocketReach_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ROCKETREACH")
+ inactiveSecret := testSecrets.MustGetField("ROCKETREACH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RocketReach,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RocketReach,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RocketReach.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RocketReach.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rocketreach/rocketreach_test.go b/pkg/detectors/rocketreach/rocketreach_test.go
index 651f1c2a8..330977e96 100644
--- a/pkg/detectors/rocketreach/rocketreach_test.go
+++ b/pkg/detectors/rocketreach/rocketreach_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package rocketreach
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRocketReach_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ROCKETREACH")
- inactiveSecret := testSecrets.MustGetField("ROCKETREACH_INACTIVE")
+var (
+ validPattern = "c897n-ee7bm86g-9vh17ixulnqwz4qmxk2bjf77"
+ invalidPattern = "c89?n-ee7bm86g-9vh17ixulnqwz4qmxk2bjf77"
+ keyword = "rocketreach"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRocketReach_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RocketReach,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rocketreach",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rocketreach secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RocketReach,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RocketReach.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RocketReach.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rockset/rockset_integration_test.go b/pkg/detectors/rockset/rockset_integration_test.go
new file mode 100644
index 000000000..e00474186
--- /dev/null
+++ b/pkg/detectors/rockset/rockset_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rockset
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRockset_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ROCKSET")
+ inactiveSecret := testSecrets.MustGetField("ROCKSET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rockset secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rockset,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rockset secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rockset,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Rockset.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Rockset.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rockset/rockset_test.go b/pkg/detectors/rockset/rockset_test.go
index 4d18254b1..36e84755e 100644
--- a/pkg/detectors/rockset/rockset_test.go
+++ b/pkg/detectors/rockset/rockset_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package rockset
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRockset_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ROCKSET")
- inactiveSecret := testSecrets.MustGetField("ROCKSET_INACTIVE")
+var (
+ validPattern = "rryYLy3RUXs72dcg6ydXOVWVr6KTeAym0RviPHcIPGafLdzyth0cKelCBa0oJC1q"
+ invalidPattern = "rryYLy3RUXs72dcg6ydXOVWVr6KTeAym?RviPHcIPGafLdzyth0cKelCBa0oJC1q"
+ keyword = "rockset"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRockset_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rockset secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rockset,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rockset",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rockset secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rockset,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Rockset.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Rockset.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/roninapp/roninapp_integration_test.go b/pkg/detectors/roninapp/roninapp_integration_test.go
new file mode 100644
index 000000000..090603947
--- /dev/null
+++ b/pkg/detectors/roninapp/roninapp_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package roninapp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRoninApp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RONINAPP")
+ inactiveSecret := testSecrets.MustGetField("RONINAPP_INACTIVE")
+ domain := testSecrets.MustGetField("RONINAPP_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RoninApp,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RoninApp,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RoninApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RoninApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/roninapp/roninapp_test.go b/pkg/detectors/roninapp/roninapp_test.go
index 67a47781e..34eff264c 100644
--- a/pkg/detectors/roninapp/roninapp_test.go
+++ b/pkg/detectors/roninapp/roninapp_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package roninapp
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRoninApp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RONINAPP")
- inactiveSecret := testSecrets.MustGetField("RONINAPP_INACTIVE")
- domain := testSecrets.MustGetField("RONINAPP_DOMAIN")
+var (
+ validKey = "tmvSxOOxP32WgoPfF2tWEmtsq8"
+ invalidKey = "tmvSxOOxP32Wg?PfF2tWEmtsq8"
+ validId = "XEZOhulA1oP7vPSc4K"
+ invalidId = "?EZOhulA1oP7vPSc4?"
+ keyword = "roninapp"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRoninApp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s", secret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RoninApp,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword roninapp",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey, validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a roninapp secret %s within roninappdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RoninApp,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RoninApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RoninApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/route4me/route4me_integration_test.go b/pkg/detectors/route4me/route4me_integration_test.go
new file mode 100644
index 000000000..4f580fd25
--- /dev/null
+++ b/pkg/detectors/route4me/route4me_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package route4me
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRoute4me_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ROUTE4ME")
+ inactiveSecret := testSecrets.MustGetField("ROUTE4ME_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a route4me secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Route4me,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a route4me secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Route4me,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Route4me.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Route4me.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/route4me/route4me_test.go b/pkg/detectors/route4me/route4me_test.go
index de2b1e24e..24e8558fd 100644
--- a/pkg/detectors/route4me/route4me_test.go
+++ b/pkg/detectors/route4me/route4me_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package route4me
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRoute4me_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ROUTE4ME")
- inactiveSecret := testSecrets.MustGetField("ROUTE4ME_INACTIVE")
+var (
+ validPattern = "WLCO3GIZN6QD2OO9DRNLM4PD7PMH7WH9"
+ invalidPattern = "WLCO3GIZN6Q?2OO9DRNLM4PD7PMH7WH9"
+ keyword = "route4me"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRoute4me_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a route4me secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Route4me,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword route4me",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a route4me secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Route4me,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Route4me.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Route4me.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/rownd/rownd_integration_test.go b/pkg/detectors/rownd/rownd_integration_test.go
new file mode 100644
index 000000000..1fbb340f1
--- /dev/null
+++ b/pkg/detectors/rownd/rownd_integration_test.go
@@ -0,0 +1,119 @@
+//go:build detectors
+// +build detectors
+
+package rownd
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRownd_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("ROWND_APP_ID")
+ key := testSecrets.MustGetField("ROWND")
+ secret := testSecrets.MustGetField("ROWND_SECRET")
+ inactiveSecret := testSecrets.MustGetField("ROWND_SECRET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s", secret, id, key)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rownd,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s but not valid", inactiveSecret, id, key)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Rownd,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Rownd.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Rownd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rownd/rownd_test.go b/pkg/detectors/rownd/rownd_test.go
index 1fbb340f1..3fe7b34a7 100644
--- a/pkg/detectors/rownd/rownd_test.go
+++ b/pkg/detectors/rownd/rownd_test.go
@@ -1,118 +1,84 @@
-//go:build detectors
-// +build detectors
-
package rownd
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRownd_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("ROWND_APP_ID")
- key := testSecrets.MustGetField("ROWND")
- secret := testSecrets.MustGetField("ROWND_SECRET")
- inactiveSecret := testSecrets.MustGetField("ROWND_SECRET_INACTIVE")
+var (
+ validKey = "rvhpdb59-6ni1-60q3-jqng-orz5edgkq333"
+ invalidKey = "rvhpdb59?6ni1-60q3-jqng-orz5edgkq333"
+ validSecret = "uuzywot3x1nlfeogbyft1r9ojb234smw0fnmn7s8kj1s1cc1"
+ invalidSecret = "uuzywot3x1nlfeogbyft1r9o?b234smw0fnmn7s8kj1s1cc1"
+ validId = "174079606475951818"
+ invalidId = "17?079606475951818"
+ keyword = "rownd"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRownd_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s", secret, id, key)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rownd,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rownd",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret, keyword, validId),
+ want: []string{validKey + validSecret},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rownd secret %s within rownd id %s and rownd key %s but not valid", inactiveSecret, id, key)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Rownd,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Rownd.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Rownd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/rubygems/rubygems_integration_test.go b/pkg/detectors/rubygems/rubygems_integration_test.go
new file mode 100644
index 000000000..bbb51c57f
--- /dev/null
+++ b/pkg/detectors/rubygems/rubygems_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package rubygems
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRubyGems_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RUBYGEMS_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("RUBYGEMS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rubygems secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RubyGems,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a rubygems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RubyGems,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RubyGems.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RubyGems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/rubygems/rubygems_test.go b/pkg/detectors/rubygems/rubygems_test.go
index aa681f83b..eb002e731 100644
--- a/pkg/detectors/rubygems/rubygems_test.go
+++ b/pkg/detectors/rubygems/rubygems_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package rubygems
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRubyGems_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RUBYGEMS_TOKEN")
- inactiveSecret := testSecrets.MustGetField("RUBYGEMS_INACTIVE")
+var (
+ validPattern = "rubygems_6gbxymmsz3xnk5dg1jc8lm9xekr652c1hr0xt5dp33pn8olh"
+ invalidPattern = "rubygems_6gbxymmsz3xnk5dg1jc?lm9xekr652c1hr0xt5dp33pn8olh"
+ keyword = "rubygems"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRubyGems_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rubygems secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RubyGems,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword rubygems",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a rubygems secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RubyGems,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RubyGems.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RubyGems.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/runrunit/runrunit_integration_test.go b/pkg/detectors/runrunit/runrunit_integration_test.go
new file mode 100644
index 000000000..a847f8f38
--- /dev/null
+++ b/pkg/detectors/runrunit/runrunit_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package runrunit
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRunRunIt_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("RUNRUNIT")
+ inactiveSecret := testSecrets.MustGetField("RUNRUNIT_INACTIVE")
+ userToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN")
+ inactiveUserToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within", secret, userToken)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RunRunIt,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within but not valid", inactiveSecret, inactiveUserToken)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_RunRunIt,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("RunRunIt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("RunRunIt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/runrunit/runrunit_test.go b/pkg/detectors/runrunit/runrunit_test.go
index 8cdeabd2e..86a42592d 100644
--- a/pkg/detectors/runrunit/runrunit_test.go
+++ b/pkg/detectors/runrunit/runrunit_test.go
@@ -1,122 +1,83 @@
-//go:build detectors
-// +build detectors
-
package runrunit
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestRunRunIt_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("RUNRUNIT")
- inactiveSecret := testSecrets.MustGetField("RUNRUNIT_INACTIVE")
- userToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN")
- inactiveUserToken := testSecrets.MustGetField("RUNRUNIT_USERTOKEN_INACTIVE")
+var (
+ validKey = "fdb658d66e09f2caaf9d452e544c3253"
+ invalidKey = "fdb658d66e09f2c?af9d452e544c3253"
+ validUserToken = "iUm7PtHOhdZxnn4jb5YR"
+ invalidUserToken = "iUm7PtHOhd?xnn4jb5YR"
+ keyword = "runrunit"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestRunRunIt_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within", secret, userToken)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RunRunIt,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword runrunit",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validUserToken),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a runrunit secret %s with runrunit user token %s within but not valid", inactiveSecret, inactiveUserToken)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_RunRunIt,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidUserToken),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("RunRunIt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("RunRunIt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/saladcloudapikey/saladcloudapikey_integration_test.go b/pkg/detectors/saladcloudapikey/saladcloudapikey_integration_test.go
new file mode 100644
index 000000000..2a3ccda20
--- /dev/null
+++ b/pkg/detectors/saladcloudapikey/saladcloudapikey_integration_test.go
@@ -0,0 +1,161 @@
+//go:build detectors
+// +build detectors
+
+package saladcloudapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSaladCloudApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SALADCLOUDAPIKEY")
+ inactiveSecret := testSecrets.MustGetField("SALADCLOUDAPIKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SaladCloudApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("SaladCloudApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go b/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go
index 95cadb185..8612ca7c8 100644
--- a/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go
+++ b/pkg/detectors/saladcloudapikey/saladcloudapikey_test.go
@@ -1,21 +1,13 @@
-//go:build detectors
-// +build detectors
-
package saladcloudapikey
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestSaladCloudApiKey_Pattern(t *testing.T) {
@@ -80,146 +72,3 @@ func TestSaladCloudApiKey_Pattern(t *testing.T) {
})
}
}
-
-func TestSaladCloudApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SALADCLOUDAPIKEY")
- inactiveSecret := testSecrets.MustGetField("SALADCLOUDAPIKEY_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a SaladCloud API Key secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SaladCloudApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SaladCloudApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
- }
- }
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("SaladCloudApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
- }
- })
- }
-}
diff --git a/pkg/detectors/salesblink/salesblink_integration_test.go b/pkg/detectors/salesblink/salesblink_integration_test.go
new file mode 100644
index 000000000..b620ebd9c
--- /dev/null
+++ b/pkg/detectors/salesblink/salesblink_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package salesblink
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSalesblink_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SALESBLINK_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SALESBLINK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesblink secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesblink,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesblink secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesblink,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Salesblink.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Salesblink.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/salesblink/salesblink_test.go b/pkg/detectors/salesblink/salesblink_test.go
index 43b662ccc..0b2a16161 100644
--- a/pkg/detectors/salesblink/salesblink_test.go
+++ b/pkg/detectors/salesblink/salesblink_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package salesblink
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSalesblink_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SALESBLINK_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SALESBLINK_INACTIVE")
+var (
+ validPattern = "key-8DbqWq7r9XKxhUqp3THfA1VMxggreCczfejJsT3UFGISAcEGrrhiA6Ju453dMUXB"
+ invalidPattern = "key-8DbqWq7r9XKxhUqp3THfA1VMxggreC?zfejJsT3UFGISAcEGrrhiA6Ju453dMUXB"
+ keyword = "salesblink"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSalesblink_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesblink secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesblink,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword salesblink",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesblink secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesblink,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Salesblink.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Salesblink.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/salescookie/salescookie_integration_test.go b/pkg/detectors/salescookie/salescookie_integration_test.go
new file mode 100644
index 000000000..66bdccb03
--- /dev/null
+++ b/pkg/detectors/salescookie/salescookie_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package salescookie
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSalescookie_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SALESCOOKIE")
+ inactiveSecret := testSecrets.MustGetField("SALESCOOKIE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salescookie secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salescookie,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salescookie secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salescookie,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Salescookie.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Salescookie.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/salescookie/salescookie_test.go b/pkg/detectors/salescookie/salescookie_test.go
index 48196acf8..70c81a2c2 100644
--- a/pkg/detectors/salescookie/salescookie_test.go
+++ b/pkg/detectors/salescookie/salescookie_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package salescookie
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSalescookie_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SALESCOOKIE")
- inactiveSecret := testSecrets.MustGetField("SALESCOOKIE_INACTIVE")
+var (
+ validPattern = "4nXYKv]mHDeom5esert30Ea7]4QEfMg6"
+ invalidPattern = "4nXYKv]mHD?om5esert30Ea7]4QEfMg6"
+ keyword = "salescookie"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSalescookie_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salescookie secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salescookie,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword salescookie",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salescookie secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salescookie,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Salescookie.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Salescookie.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/salesflare/salesflare_integration_test.go b/pkg/detectors/salesflare/salesflare_integration_test.go
new file mode 100644
index 000000000..a0a182538
--- /dev/null
+++ b/pkg/detectors/salesflare/salesflare_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package salesflare
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSalesflare_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SALESFLARE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SALESFLARE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesflare secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesflare,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesflare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesflare,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Salesflare.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Salesflare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/salesflare/salesflare_test.go b/pkg/detectors/salesflare/salesflare_test.go
index e3b0490c7..e70397b90 100644
--- a/pkg/detectors/salesflare/salesflare_test.go
+++ b/pkg/detectors/salesflare/salesflare_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package salesflare
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSalesflare_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SALESFLARE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SALESFLARE_INACTIVE")
+var (
+ validPattern = "7dgVfsiOYJhYmw5aQc9DfE6V5jLPuhN2LzsTkhn32KRaZ"
+ invalidPattern = "7dgVfsiOYJhYmw5aQc9DfE?V5jLPuhN2LzsTkhn32KRaZ"
+ keyword = "salesflare"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSalesflare_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesflare secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesflare,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword salesflare",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesflare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesflare,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Salesflare.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Salesflare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/salesforce/salesforce_integration_test.go b/pkg/detectors/salesforce/salesforce_integration_test.go
new file mode 100644
index 000000000..fd5fde6f4
--- /dev/null
+++ b/pkg/detectors/salesforce/salesforce_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package salesforce
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSalesforce_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ instance := testSecrets.MustGetField("SALESFORCE_INSTANCE")
+ // Salesforce secrets are not valid for long, so we may need to regenerate them.
+ // Steps to regenerate:
+ // 1. Install Salesforce CLI
+ // 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault.
+ // 3. Run `sfdx org display --targetusername [email protected]`to obtain new token.
+ // Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm
+ token := testSecrets.MustGetField("SALESFORCE_TOKEN")
+ inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesforce,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesforce,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesforce,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesforce,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/salesforce/salesforce_test.go b/pkg/detectors/salesforce/salesforce_test.go
index 6c7721e97..99da35666 100644
--- a/pkg/detectors/salesforce/salesforce_test.go
+++ b/pkg/detectors/salesforce/salesforce_test.go
@@ -1,168 +1,82 @@
-//go:build detectors
-// +build detectors
-
package salesforce
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSalesforce_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- instance := testSecrets.MustGetField("SALESFORCE_INSTANCE")
- // Salesforce secrets are not valid for long, so we may need to regenerate them.
- // Steps to regenerate:
- // 1. Install Salesforce CLI
- // 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault.
- // 3. Run `sfdx org display --targetusername [email protected]`to obtain new token.
- // Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm
- token := testSecrets.MustGetField("SALESFORCE_TOKEN")
- inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN")
+var (
+ validAccessToken = "00MMW9rRmrlVvBi!gZTTPUfk_nkghR_01qsw3NdcHTeDF8dQKWJmiOcfSjfdymrqXH0_vMci6VmxHzrlw07JSfeyMJrA_N89fUJU9vrAYWn_isTE"
+ invalidAccessToken = "00MMW9rRmrlVvBi!gZTTPUfk_nkghR_01qsw3NdcHTeDF8dQKWJmiOcf?jfdymrqXH0_vMci6VmxHzrlw07JSfeyMJrA_N89fUJU9vrAYWn_isTE"
+ validInstance = "https://wDIMT.HmGz15hePYJBiaiG4leH6y.my.salesforce.com"
+ invalidInstance = "https://wDIMT.HmGz15hePYJBi?iG4leH6y.my.salesforce.com"
+ keyword = "salesforce"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSalesforce_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesforce,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesforce,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesforce,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - with keyword salesforce",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validAccessToken, keyword, validInstance),
+ want: []string{validAccessToken},
},
{
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesforce,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidAccessToken, keyword, invalidInstance),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
+ return
+ }
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/salesmate/salesmate_integration_test.go b/pkg/detectors/salesmate/salesmate_integration_test.go
new file mode 100644
index 000000000..c1bc3d341
--- /dev/null
+++ b/pkg/detectors/salesmate/salesmate_integration_test.go
@@ -0,0 +1,118 @@
+package salesmate
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSalesmate_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SALESMATE")
+ inactiveSecret := testSecrets.MustGetField("SALESMATE_INACTIVE")
+ domain := testSecrets.MustGetField("SALESMATE_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesmate secret %s within salesmateDomain %s", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesmate,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a salesmate secret %s within but not valid salesmateDomain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Salesmate,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Salesmate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Salesmate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/salesmate/salesmate_test.go b/pkg/detectors/salesmate/salesmate_test.go
index d440be9aa..44be1f026 100644
--- a/pkg/detectors/salesmate/salesmate_test.go
+++ b/pkg/detectors/salesmate/salesmate_test.go
@@ -4,115 +4,80 @@ import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSalesmate_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SALESMATE")
- inactiveSecret := testSecrets.MustGetField("SALESMATE_INACTIVE")
- domain := testSecrets.MustGetField("SALESMATE_DOMAIN")
+var (
+ validKey = "398A3699-a4ae-5e7d-ae08-48ef223f1c45"
+ invalidKey = "398A3699?a4ae-5e7d-ae08-48ef223f1c45"
+ validDomain = "dte2oxDZZSbPhyX1"
+ invalidDomain = "?te2oxDZZSbPhyX?"
+ keyword = "salesmate"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSalesmate_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesmate secret %s within salesmateDomain %s", secret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesmate,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword salesmate",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),
+ want: []string{validKey, validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a salesmate secret %s within but not valid salesmateDomain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Salesmate,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Salesmate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Salesmate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/saucelabs/saucelabs_integration_test.go b/pkg/detectors/saucelabs/saucelabs_integration_test.go
new file mode 100644
index 000000000..7145d0f5d
--- /dev/null
+++ b/pkg/detectors/saucelabs/saucelabs_integration_test.go
@@ -0,0 +1,140 @@
+//go:build detectors
+// +build detectors
+
+package saucelabs
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSauceLabs_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("SAUCELABS")
+ secret := testSecrets.MustGetField("SAUCELABS_SECRET")
+ inactiveSecret := testSecrets.MustGetField("SAUCELABS_SECRET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(createFakeString(id, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SauceLabs,
+ Verified: true,
+ RawV2: []byte(id + secret),
+ },
+ {
+ DetectorType: detectorspb.DetectorType_SauceLabs,
+ Verified: false,
+ RawV2: []byte(secret + secret),
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("username: %s\n saucelabskey: %s", id, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SauceLabs,
+ Verified: false,
+ RawV2: []byte(id + inactiveSecret),
+ },
+ {
+ DetectorType: detectorspb.DetectorType_SauceLabs,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + inactiveSecret),
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SauceLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SauceLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+func createFakeString(username, key string) string {
+ return `
+ username: ` + username + `
+ saucelabskey: ` + key + `
+ `
+}
diff --git a/pkg/detectors/saucelabs/saucelabs_test.go b/pkg/detectors/saucelabs/saucelabs_test.go
index 7145d0f5d..533feb215 100644
--- a/pkg/detectors/saucelabs/saucelabs_test.go
+++ b/pkg/detectors/saucelabs/saucelabs_test.go
@@ -1,140 +1,85 @@
-//go:build detectors
-// +build detectors
-
package saucelabs
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSauceLabs_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("SAUCELABS")
- secret := testSecrets.MustGetField("SAUCELABS_SECRET")
- inactiveSecret := testSecrets.MustGetField("SAUCELABS_SECRET_INACTIVE")
+var (
+ validUsername = "nBpeh2YF3tjMNmKAUHrDpfXJZJ.ZUhWmObRC735zdh99LeJh4Kcr_2"
+ invalidUsername = "?Bpeh?YF3tjMNmKAUHrDpfXJZJ.ZUhWmObRC735zdh99LeJh4Kcr_2"
+ validKey = "9toqmh6r-g2ly-k3fr-fiyr-5jnnewlxy3u9"
+ invalidKey = "9toqmh6r?g2ly-k3fr-fiyr-5jnnewlxy3u9"
+ validBaseUrl = "api.eu-west-0,saucelabs.com"
+ invalidBaseUrl = "?pi.eu-west-0,saucelabs.co?"
+ keyword = "saucelabs"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSauceLabs_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(createFakeString(id, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SauceLabs,
- Verified: true,
- RawV2: []byte(id + secret),
- },
- {
- DetectorType: detectorspb.DetectorType_SauceLabs,
- Verified: false,
- RawV2: []byte(secret + secret),
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("username: %s\n saucelabskey: %s", id, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SauceLabs,
- Verified: false,
- RawV2: []byte(id + inactiveSecret),
- },
- {
- DetectorType: detectorspb.DetectorType_SauceLabs,
- Verified: false,
- RawV2: []byte(inactiveSecret + inactiveSecret),
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword saucelabs",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validUsername, keyword, validKey, keyword, validBaseUrl),
+ want: []string{".com" + validKey, "token" + validKey},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidUsername, keyword, invalidKey, keyword, invalidBaseUrl),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SauceLabs.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SauceLabs.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
-
-func createFakeString(username, key string) string {
- return `
- username: ` + username + `
- saucelabskey: ` + key + `
- `
-}
diff --git a/pkg/detectors/scalewaykey/scalewaykey_integration_test.go b/pkg/detectors/scalewaykey/scalewaykey_integration_test.go
new file mode 100644
index 000000000..409b92dcd
--- /dev/null
+++ b/pkg/detectors/scalewaykey/scalewaykey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scalewaykey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScalewayKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCALEWAYKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SCALEWAYKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScalewayKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScalewayKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScalewayKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScalewayKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scalewaykey/scalewaykey_test.go b/pkg/detectors/scalewaykey/scalewaykey_test.go
index 58bbd79a0..5b4f37481 100644
--- a/pkg/detectors/scalewaykey/scalewaykey_test.go
+++ b/pkg/detectors/scalewaykey/scalewaykey_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scalewaykey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScalewayKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCALEWAYKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SCALEWAYKEY_INACTIVE")
+var (
+ validPattern = "lj6z1t7m-hdu4-647t-uoju-nlfnizsv7zry"
+ invalidPattern = "lj6z1t7m?hdu4-647t-uoju-nlfnizsv7zry"
+ keyword = "scalewaykey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScalewayKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScalewayKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scalewaykey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scalewaykey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScalewayKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScalewayKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScalewayKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scalr/scalr_integration_test.go b/pkg/detectors/scalr/scalr_integration_test.go
new file mode 100644
index 000000000..e1103d10f
--- /dev/null
+++ b/pkg/detectors/scalr/scalr_integration_test.go
@@ -0,0 +1,126 @@
+package scalr
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScalr_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCALR")
+ inactiveSecret := testSecrets.MustGetField("SCALR_INACTIVE")
+ id := testSecrets.MustGetField("SCALR_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scalr secret %s within scalr %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Scalr,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Scalr,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scalr secret %s within but not valid scalr %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Scalr,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Scalr,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Scalr.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Scalr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scalr/scalr_test.go b/pkg/detectors/scalr/scalr_test.go
index 2fd07bf70..cba85fa0b 100644
--- a/pkg/detectors/scalr/scalr_test.go
+++ b/pkg/detectors/scalr/scalr_test.go
@@ -4,123 +4,80 @@ import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScalr_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCALR")
- inactiveSecret := testSecrets.MustGetField("SCALR_INACTIVE")
- id := testSecrets.MustGetField("SCALR_DOMAIN")
+var (
+ validKey = "bmy2Q33nZnlwZ5wCu_umGbBHF_6547oPzvAGLpUEwhWwOJmU5DUlZIOWZ0Yr7g2Yt4s3ZyyfWxzOlLLOmkGXpzuQ3.gEhgGUHUngyyiPIZjVoYQ.W0O8k4G3JBC705AXkXxANwrF"
+ invalidKey = "bmy2Q33nZnlwZ5wCu?umGbBHF?6547oPzvAGLpUEwhWwOJmU5DUlZIOWZ0Yr7g2Yt4s3ZyyfWxzOlLLOmkGXpzuQ3?gEhgGUHUngyyiPIZjVoYQ.W0O8k4G3JBC705AXkXxANwrF"
+ validId = "h7bcggvlzhq6t5m47f37jav"
+ invalidId = "?7bcgg?lzhq6t5m47f37jav"
+ keyword = "scalr"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScalr_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scalr secret %s within scalr %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Scalr,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Scalr,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scalr",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey, validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scalr secret %s within but not valid scalr %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Scalr,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Scalr,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Scalr.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Scalr.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scrapeowl/scrapeowl_integration_test.go b/pkg/detectors/scrapeowl/scrapeowl_integration_test.go
new file mode 100644
index 000000000..3b38277fc
--- /dev/null
+++ b/pkg/detectors/scrapeowl/scrapeowl_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scrapeowl
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScrapeowl_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPEOWL")
+ inactiveSecret := testSecrets.MustGetField("SCRAPEOWL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Scrapeowl,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Scrapeowl,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Scrapeowl.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Scrapeowl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scrapeowl/scrapeowl_test.go b/pkg/detectors/scrapeowl/scrapeowl_test.go
index 519f9a03c..1d05ef18f 100644
--- a/pkg/detectors/scrapeowl/scrapeowl_test.go
+++ b/pkg/detectors/scrapeowl/scrapeowl_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scrapeowl
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScrapeowl_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPEOWL")
- inactiveSecret := testSecrets.MustGetField("SCRAPEOWL_INACTIVE")
+var (
+ validPattern = "dwkiwm366z8gqutb1ukv9sztut04be"
+ invalidPattern = "dwkiwm366z8gqut?1ukv9sztut04be"
+ keyword = "scrapeowl"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScrapeowl_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Scrapeowl,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scrapeowl",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapeowl secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Scrapeowl,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Scrapeowl.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Scrapeowl.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scraperapi/scraperapi_integration_test.go b/pkg/detectors/scraperapi/scraperapi_integration_test.go
new file mode 100644
index 000000000..8b8cec04d
--- /dev/null
+++ b/pkg/detectors/scraperapi/scraperapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scraperapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScraperAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPERAPI")
+ inactiveSecret := testSecrets.MustGetField("SCRAPERAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScraperAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScraperAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScraperAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScraperAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scraperapi/scraperapi_test.go b/pkg/detectors/scraperapi/scraperapi_test.go
index c809f33d8..100c82aca 100644
--- a/pkg/detectors/scraperapi/scraperapi_test.go
+++ b/pkg/detectors/scraperapi/scraperapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scraperapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScraperAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPERAPI")
- inactiveSecret := testSecrets.MustGetField("SCRAPERAPI_INACTIVE")
+var (
+ validPattern = "28ksnqsrn7q12wp11onrmr4bh1dvi7ut"
+ invalidPattern = "28ksnqsrn7q?2wp11onrmr4bh1dvi7ut"
+ keyword = "scraperapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScraperAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScraperAPI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scraperapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scraperapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScraperAPI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScraperAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScraperAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scraperbox/scraperbox_integration_test.go b/pkg/detectors/scraperbox/scraperbox_integration_test.go
new file mode 100644
index 000000000..22234582c
--- /dev/null
+++ b/pkg/detectors/scraperbox/scraperbox_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scraperbox
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScraperBox_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPERBOX")
+ inactiveSecret := testSecrets.MustGetField("SCRAPERBOX_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScraperBox,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScraperBox,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScraperBox.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScraperBox.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scraperbox/scraperbox_test.go b/pkg/detectors/scraperbox/scraperbox_test.go
index 4a867efbf..ef7c06dfd 100644
--- a/pkg/detectors/scraperbox/scraperbox_test.go
+++ b/pkg/detectors/scraperbox/scraperbox_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scraperbox
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScraperBox_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPERBOX")
- inactiveSecret := testSecrets.MustGetField("SCRAPERBOX_INACTIVE")
+var (
+ validPattern = "H8AOZKMTK849QUN2CAKZ8L77YS53I34L"
+ invalidPattern = "H8AOZKMTK849QUN2?AKZ8L77YS53I34L"
+ keyword = "scraperbox"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScraperBox_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScraperBox,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scraperbox",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scraperbox secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScraperBox,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScraperBox.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScraperBox.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scrapestack/scrapestack_integration_test.go b/pkg/detectors/scrapestack/scrapestack_integration_test.go
new file mode 100644
index 000000000..5560c56f8
--- /dev/null
+++ b/pkg/detectors/scrapestack/scrapestack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scrapestack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScrapeStack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPESTACK")
+ inactiveSecret := testSecrets.MustGetField("SCRAPESTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrapeStack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrapeStack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScrapeStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScrapeStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scrapestack/scrapestack_test.go b/pkg/detectors/scrapestack/scrapestack_test.go
index 0fdb02558..a46fa6c9c 100644
--- a/pkg/detectors/scrapestack/scrapestack_test.go
+++ b/pkg/detectors/scrapestack/scrapestack_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scrapestack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScrapeStack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPESTACK")
- inactiveSecret := testSecrets.MustGetField("SCRAPESTACK_INACTIVE")
+var (
+ validPattern = "gdozp26v2hzq91fh8k412n5ik5klj59p"
+ invalidPattern = "gdozp26v2hzq91fh?k412n5ik5klj59p"
+ keyword = "scrapestack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScrapeStack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrapeStack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scrapestack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapestack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrapeStack,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScrapeStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScrapeStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scrapfly/scrapfly_integration_test.go b/pkg/detectors/scrapfly/scrapfly_integration_test.go
new file mode 100644
index 000000000..1634b6583
--- /dev/null
+++ b/pkg/detectors/scrapfly/scrapfly_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scrapfly
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScrapfly_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPFLY")
+ inactiveSecret := testSecrets.MustGetField("SCRAPFLY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Scrapfly,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Scrapfly,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Scrapfly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Scrapfly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scrapfly/scrapfly_test.go b/pkg/detectors/scrapfly/scrapfly_test.go
index 34a7ff5b0..71a844edc 100644
--- a/pkg/detectors/scrapfly/scrapfly_test.go
+++ b/pkg/detectors/scrapfly/scrapfly_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scrapfly
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScrapfly_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPFLY")
- inactiveSecret := testSecrets.MustGetField("SCRAPFLY_INACTIVE")
+var (
+ validPattern = "ajw59xcg6r6uywscubwxh5t3qmji2tp2"
+ invalidPattern = "ajw59xcg6r6?ywscubwxh5t3qmji2tp2"
+ keyword = "scrapfly"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScrapfly_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Scrapfly,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scrapfly",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapfly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Scrapfly,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Scrapfly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Scrapfly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scrapingant/scrapingant_integration_test.go b/pkg/detectors/scrapingant/scrapingant_integration_test.go
new file mode 100644
index 000000000..eb69636ca
--- /dev/null
+++ b/pkg/detectors/scrapingant/scrapingant_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scrapingant
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScrapingAnt_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPINGANT")
+ inactiveSecret := testSecrets.MustGetField("SCRAPINGANT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrapingAnt,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrapingAnt,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScrapingAnt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScrapingAnt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scrapingant/scrapingant_test.go b/pkg/detectors/scrapingant/scrapingant_test.go
index eb69636ca..dcce62fe9 100644
--- a/pkg/detectors/scrapingant/scrapingant_test.go
+++ b/pkg/detectors/scrapingant/scrapingant_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scrapingant
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScrapingAnt_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPINGANT")
- inactiveSecret := testSecrets.MustGetField("SCRAPINGANT_INACTIVE")
+var (
+ validPattern = "uifssyi7y40rf7ryyr3vczj8xbhbv49f"
+ invalidPattern = "uifss?i7y40rf7ryyr3vczj8xbhbv49f"
+ keyword = "scrapingant"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScrapingAnt_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrapingAnt,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scrapingant",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapingant secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrapingAnt,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScrapingAnt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScrapingAnt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scrapingbee/scrapingbee_integration_test.go b/pkg/detectors/scrapingbee/scrapingbee_integration_test.go
new file mode 100644
index 000000000..2fb482ed2
--- /dev/null
+++ b/pkg/detectors/scrapingbee/scrapingbee_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scrapingbee
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScrapingBee_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRAPINGBEE")
+ inactiveSecret := testSecrets.MustGetField("SCRAPINGBEE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrapingBee,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrapingBee,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScrapingBee.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScrapingBee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scrapingbee/scrapingbee_test.go b/pkg/detectors/scrapingbee/scrapingbee_test.go
index 561c67eb2..24061f0a4 100644
--- a/pkg/detectors/scrapingbee/scrapingbee_test.go
+++ b/pkg/detectors/scrapingbee/scrapingbee_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scrapingbee
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScrapingBee_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPINGBEE")
- inactiveSecret := testSecrets.MustGetField("SCRAPINGBEE_INACTIVE")
+var (
+ validPattern = "HOLTCTPVL3V35NK795FY6INWQKTTTSGL6W26JKHLC3CGUNJ1DV35SWPUAZ26BUX2EEZZ7T2SJBOL8WZB"
+ invalidPattern = "HOLT?TPVL3V35NK795FY6INWQKTTTSGL6W26JKHLC3CGUNJ1DV35SWPUAZ26BUX2EEZZ7T2SJBOL8WZB"
+ keyword = "scrapingbee"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScrapingBee_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrapingBee,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scrapingbee",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapingbee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrapingBee,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScrapingBee.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScrapingBee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/screenshotapi/screenshotapi_integration_test.go b/pkg/detectors/screenshotapi/screenshotapi_integration_test.go
new file mode 100644
index 000000000..61cfe00e9
--- /dev/null
+++ b/pkg/detectors/screenshotapi/screenshotapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package screenshotapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScreenshotAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCREENSHOTAPI")
+ inactiveSecret := testSecrets.MustGetField("SCREENSHOTAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScreenshotAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScreenshotAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/screenshotapi/screenshotapi_test.go b/pkg/detectors/screenshotapi/screenshotapi_test.go
index 34601e49b..2c028fdd5 100644
--- a/pkg/detectors/screenshotapi/screenshotapi_test.go
+++ b/pkg/detectors/screenshotapi/screenshotapi_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package screenshotapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScreenshotAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCREENSHOTAPI")
- inactiveSecret := testSecrets.MustGetField("SCREENSHOTAPI_INACTIVE")
+var (
+ validPattern = "5QCXUYE-B9LNS58-X62XLXO-CHZJZDO"
+ invalidPattern = "5QCXUYE?B9LNS58-X62XLXO-CHZJZDO"
+ keyword = "screenshotapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScreenshotAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScreenshotAPI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword screenshotapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a screenshotapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScreenshotAPI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScreenshotAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScreenshotAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/screenshotlayer/screenshotlayer_integration_test.go b/pkg/detectors/screenshotlayer/screenshotlayer_integration_test.go
new file mode 100644
index 000000000..53a431204
--- /dev/null
+++ b/pkg/detectors/screenshotlayer/screenshotlayer_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package screenshotlayer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScreenshotLayer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCREENSHOTLAYER")
+ inactiveSecret := testSecrets.MustGetField("SCREENSHOTLAYER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScreenshotLayer,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScreenshotLayer,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScreenshotLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScreenshotLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/screenshotlayer/screenshotlayer_test.go b/pkg/detectors/screenshotlayer/screenshotlayer_test.go
index 15f03a2d7..7f7538bbc 100644
--- a/pkg/detectors/screenshotlayer/screenshotlayer_test.go
+++ b/pkg/detectors/screenshotlayer/screenshotlayer_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package screenshotlayer
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScreenshotLayer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCREENSHOTLAYER")
- inactiveSecret := testSecrets.MustGetField("SCREENSHOTLAYER_INACTIVE")
+var (
+ validPattern = "mbACjpi8P_c_6y2WMu4dUfvnvKWuDb_3"
+ invalidPattern = "mbACjpi8P_c_6y2W?u4dUfvnvKWuDb_3"
+ keyword = "screenshotlayer"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScreenshotLayer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScreenshotLayer,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword screenshotlayer",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a screenshotlayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScreenshotLayer,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScreenshotLayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScreenshotLayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/scrutinizerci/scrutinizerci_integration_test.go b/pkg/detectors/scrutinizerci/scrutinizerci_integration_test.go
new file mode 100644
index 000000000..eaf8439f9
--- /dev/null
+++ b/pkg/detectors/scrutinizerci/scrutinizerci_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package scrutinizerci
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestScrutinizerCi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SCRUTINIZERCI")
+ inactiveSecret := testSecrets.MustGetField("SCRUTINIZERCI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrutinizerCi,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ScrutinizerCi,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ScrutinizerCi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ScrutinizerCi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/scrutinizerci/scrutinizerci_test.go b/pkg/detectors/scrutinizerci/scrutinizerci_test.go
index f43c24e8e..e4798ea40 100644
--- a/pkg/detectors/scrutinizerci/scrutinizerci_test.go
+++ b/pkg/detectors/scrutinizerci/scrutinizerci_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package scrutinizerci
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestScrutinizerCi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRUTINIZERCI")
- inactiveSecret := testSecrets.MustGetField("SCRUTINIZERCI_INACTIVE")
+var (
+ validPattern = "tj8dzw8on7hw5j5wtiphzc4hv5wfvb2kbk4djyz8r1oxqes8otd3ms0mrqw8jlwt"
+ invalidPattern = "tj8dzw8on7hw5j5wtiphzc4hv5wfv?2kbk4djyz8r1oxqes8otd3ms0mrqw8jlwt"
+ keyword = "scrutinizerci"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestScrutinizerCi_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrutinizerCi,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword scrutinizerci",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrutinizerci secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScrutinizerCi,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScrutinizerCi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScrutinizerCi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/securitytrails/securitytrails_integration_test.go b/pkg/detectors/securitytrails/securitytrails_integration_test.go
new file mode 100644
index 000000000..73f9bccc5
--- /dev/null
+++ b/pkg/detectors/securitytrails/securitytrails_integration_test.go
@@ -0,0 +1,136 @@
+//go:build detectors
+// +build detectors
+
+package securitytrails
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSecurityTrails_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SECURITYTRAILS")
+ inactiveSecret := testSecrets.MustGetField("SECURITYTRAILS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a securitytrails secret\n %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SecurityTrails,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, verified inline",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SecurityTrails,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SecurityTrails,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SecurityTrails.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SecurityTrails.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/securitytrails/securitytrails_test.go b/pkg/detectors/securitytrails/securitytrails_test.go
index 952023720..83dd8f0a1 100644
--- a/pkg/detectors/securitytrails/securitytrails_test.go
+++ b/pkg/detectors/securitytrails/securitytrails_test.go
@@ -1,136 +1,91 @@
-//go:build detectors
-// +build detectors
-
package securitytrails
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSecurityTrails_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SECURITYTRAILS")
- inactiveSecret := testSecrets.MustGetField("SECURITYTRAILS_INACTIVE")
+var (
+ validPattern = "wZ8KxGUFd2G1IrazudQo9UpGyqr8jji6"
+ invalidPattern = "wZ8KxGUFd2G1Iraz?dQo9UpGyqr8jji6"
+ keyword = "securitytrails"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSecurityTrails_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a securitytrails secret\n %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SecurityTrails,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword securitytrails",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, verified inline",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SecurityTrails,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a securitytrails secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SecurityTrails,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SecurityTrails.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SecurityTrails.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/segmentapikey/segmentapikey_integration_test.go b/pkg/detectors/segmentapikey/segmentapikey_integration_test.go
new file mode 100644
index 000000000..9f34cb1bb
--- /dev/null
+++ b/pkg/detectors/segmentapikey/segmentapikey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package segmentapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSegmentApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SEGMENTAPIKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SEGMENTAPIKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SegmentApiKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SegmentApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SegmentApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SegmentApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/segmentapikey/segmentapikey_test.go b/pkg/detectors/segmentapikey/segmentapikey_test.go
index 133f92853..9344a12b2 100644
--- a/pkg/detectors/segmentapikey/segmentapikey_test.go
+++ b/pkg/detectors/segmentapikey/segmentapikey_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package segmentapikey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSegmentApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SEGMENTAPIKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SEGMENTAPIKEY_INACTIVE")
+var (
+ validPattern = "ENTvbxIazIP8g_chwXUfz4M930vr4YArS00iBPtBADI.E7RjExdT9CXgjFO-EhFE1WQGjRflI9YSJG-cCYPUB5x"
+ invalidPattern = "ENTvbxIazIP8g_chwXUfz4M930vr4YArS00iBPtBADI?E7RjExdT9CXgjFO-EhFE1WQGjRflI9YSJG-cCYPUB5x"
+ keyword = "segmentapikey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSegmentApiKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SegmentApiKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword segmentapikey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a segmentapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SegmentApiKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SegmentApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SegmentApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/selectpdf/selectpdf_integration_test.go b/pkg/detectors/selectpdf/selectpdf_integration_test.go
new file mode 100644
index 000000000..30906236d
--- /dev/null
+++ b/pkg/detectors/selectpdf/selectpdf_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package selectpdf
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSelectPDF_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SELECTPDF")
+ inactiveSecret := testSecrets.MustGetField("SELECTPDF_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SelectPDF,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SelectPDF,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SelectPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SelectPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/selectpdf/selectpdf_test.go b/pkg/detectors/selectpdf/selectpdf_test.go
index 6757a7f1f..9da69e3dd 100644
--- a/pkg/detectors/selectpdf/selectpdf_test.go
+++ b/pkg/detectors/selectpdf/selectpdf_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package selectpdf
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSelectPDF_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SELECTPDF")
- inactiveSecret := testSecrets.MustGetField("SELECTPDF_INACTIVE")
+var (
+ validPattern = "fmajqnwjj-ia1l4in7-4r6ktkn1ikiy89x02"
+ invalidPattern = "fmajqnwjj-ia1l4in7?4r6ktkn1ikiy89x02"
+ keyword = "selectpdf"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSelectPDF_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SelectPDF,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword selectpdf",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a selectpdf secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SelectPDF,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SelectPDF.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SelectPDF.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/semaphore/semaphore_integration_test.go b/pkg/detectors/semaphore/semaphore_integration_test.go
new file mode 100644
index 000000000..c107c1ae9
--- /dev/null
+++ b/pkg/detectors/semaphore/semaphore_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package semaphore
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSemaphore_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SEMAPHORE")
+ inactiveSecret := testSecrets.MustGetField("SEMAPHORE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a semaphore secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Semaphore,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a semaphore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Semaphore,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Semaphore.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Semaphore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/semaphore/semaphore_test.go b/pkg/detectors/semaphore/semaphore_test.go
index 116535e06..60b73d46e 100644
--- a/pkg/detectors/semaphore/semaphore_test.go
+++ b/pkg/detectors/semaphore/semaphore_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package semaphore
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSemaphore_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SEMAPHORE")
- inactiveSecret := testSecrets.MustGetField("SEMAPHORE_INACTIVE")
+var (
+ validPattern = "e90jcvaioqwdzi0thnnpx59nv2k4efom"
+ invalidPattern = "e90jcvaioqwdzi0t?nnpx59nv2k4efom"
+ keyword = "semaphore.co"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSemaphore_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a semaphore secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Semaphore,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword semaphore.co",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a semaphore secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Semaphore,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Semaphore.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Semaphore.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sendbird/sendbird_integration_test.go b/pkg/detectors/sendbird/sendbird_integration_test.go
new file mode 100644
index 000000000..aae61bfb8
--- /dev/null
+++ b/pkg/detectors/sendbird/sendbird_integration_test.go
@@ -0,0 +1,193 @@
+//go:build detectors
+// +build detectors
+
+package sendbird
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSendbird_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SENDBIRD")
+ inactiveSecret := testSecrets.MustGetField("SENDBIRD_INACTIVE")
+ appId := testSecrets.MustGetField("SENDBIRD_APP_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sendbird,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within but not valid", appId, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sendbird,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sendbird,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sendbird,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected error code",
+ s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sendbird,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but failed to decode json",
+ s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sendbird,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sendbird/sendbird_test.go b/pkg/detectors/sendbird/sendbird_test.go
index b84849869..87ae80568 100644
--- a/pkg/detectors/sendbird/sendbird_test.go
+++ b/pkg/detectors/sendbird/sendbird_test.go
@@ -1,191 +1,82 @@
-//go:build detectors
-// +build detectors
-
package sendbird
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSendbird_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SENDBIRD")
- inactiveSecret := testSecrets.MustGetField("SENDBIRD_INACTIVE")
- appId := testSecrets.MustGetField("SENDBIRD_APP_ID")
+var (
+ validKey = "e9aeb40d34e6a7a9fb10e5876be968e344420b0c"
+ invalidKey = "?9aeb40d34e6a7a9fb10e5876be968e344420b0c"
+ validAppId = "FAB7BE5B-E6DD-670A-A3DC-91AF003DB677"
+ invalidAppId = "FAB7BE5B?E6DD-670A-A3DC-91AF003DB677"
+ keyword = "sendbird"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSendbird_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sendbird,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within but not valid", appId, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sendbird,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sendbird,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sendbird,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - with keyword sendbird",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validAppId),
+ want: []string{validKey},
},
{
- name: "found, verified but unexpected error code",
- s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sendbird,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but failed to decode json",
- s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sendbird,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidAppId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_integration_test.go b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_integration_test.go
new file mode 100644
index 000000000..7d1b856ad
--- /dev/null
+++ b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_integration_test.go
@@ -0,0 +1,158 @@
+//go:build detectors
+// +build detectors
+
+package sendbirdorganizationapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI")
+ inactiveSecret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go
index 7d1b856ad..aa3d17e16 100644
--- a/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go
+++ b/pkg/detectors/sendbirdorganizationapi/sendbirdorganizationapi_test.go
@@ -1,157 +1,90 @@
-//go:build detectors
-// +build detectors
-
package sendbirdorganizationapi
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI")
- inactiveSecret := testSecrets.MustGetField("SENDBIRDORGANIZATIONAPI_INACTIVE")
+var (
+ validPattern = "fd3ddab708ffd4b71f079272"
+ invalidPattern = "f?3ddab708ffd4b71f079272"
+ keyword = "sendbirdorganizationapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSendbirdOrganizationAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword sendbirdorganizationapi",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/sendgrid/sendgrid_integration_test.go b/pkg/detectors/sendgrid/sendgrid_integration_test.go
new file mode 100644
index 000000000..230cf1825
--- /dev/null
+++ b/pkg/detectors/sendgrid/sendgrid_integration_test.go
@@ -0,0 +1,152 @@
+//go:build detectors
+// +build detectors
+
+package sendgrid
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSendgrid_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SENDGRID")
+ secretInactive := testSecrets.MustGetField("SENDGRID_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendGrid,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo", "ExtraData")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sendgrid/sendgrid_test.go b/pkg/detectors/sendgrid/sendgrid_test.go
index 886f5ffb0..254a33ca2 100644
--- a/pkg/detectors/sendgrid/sendgrid_test.go
+++ b/pkg/detectors/sendgrid/sendgrid_test.go
@@ -1,150 +1,85 @@
-//go:build detectors
-// +build detectors
-
package sendgrid
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSendgrid_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SENDGRID")
- secretInactive := testSecrets.MustGetField("SENDGRID_INACTIVE")
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validPattern = "SG.ZV-VYwtHzJJW4wF8yPQk3.ZXg9c9DZuOgUcW1f2inP6SqfEsYG82zAe0wG7brZZ5OvruV-I"
+ invalidPattern = "SG.ZV-VYwtHzJJW4wF8yPQk3.ZXg9?9DZuOgUcW1f2inP6SqfEsYG82zAe0wG7brZZ5OvruV-I"
+ keyword = "sendgrid"
+)
+
+func TestSendgrid_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendGrid,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword sendgrid",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendGrid,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendGrid,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendGrid,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "AnalysisInfo", "ExtraData")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/sendinbluev2/sendinbluev2_integration_test.go b/pkg/detectors/sendinbluev2/sendinbluev2_integration_test.go
new file mode 100644
index 000000000..0e3546a77
--- /dev/null
+++ b/pkg/detectors/sendinbluev2/sendinbluev2_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package sendinbluev2
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSendinBlueV2_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SENDINBLUEV2_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SENDINBLUEV2_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendinBlueV2,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within but unverified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SendinBlueV2,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SendinBlueV2.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SendinBlueV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sendinbluev2/sendinbluev2_test.go b/pkg/detectors/sendinbluev2/sendinbluev2_test.go
index 0e3546a77..5f3886111 100644
--- a/pkg/detectors/sendinbluev2/sendinbluev2_test.go
+++ b/pkg/detectors/sendinbluev2/sendinbluev2_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package sendinbluev2
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSendinBlueV2_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SENDINBLUEV2_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SENDINBLUEV2_INACTIVE")
+var (
+ validPattern = "xkeysib-C6S1LXk_u4mw_uIss4MGmJpH8yrOwFep2aN5YLALYVpAb4buJ7uvxqYfrb3kZL5ao2JvUEFb1vRk79IXj"
+ invalidPattern = "xkeysib-C6S1LXk_u4mw_uIss4MGmJpH8yrOwFep2aN5?LALYVpAb4buJ7uvxqYfrb3kZL5ao2JvUEFb1vRk79IXj"
+ keyword = "sendinbluev2"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSendinBlueV2_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendinBlueV2,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sendinbluev2 secret %s within but unverified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SendinBlueV2,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sendinbluev2",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SendinBlueV2.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SendinBlueV2.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sentrytoken/sentrytoken_integration_test.go b/pkg/detectors/sentrytoken/sentrytoken_integration_test.go
new file mode 100644
index 000000000..34729e950
--- /dev/null
+++ b/pkg/detectors/sentrytoken/sentrytoken_integration_test.go
@@ -0,0 +1,229 @@
+//go:build detectors
+// +build detectors
+
+package sentrytoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSentryToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SENTRYTOKEN_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SENTRYTOKEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, would be verified but for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found and valid but unexpected api response",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, good key but wrong scope",
+ s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, account deactivated",
+ s: Scanner{client: common.ConstantResponseHttpClient(200, responseAccountDeactivated)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, account deactivated",
+ s: Scanner{client: common.ConstantResponseHttpClient(200, responseEmpty)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SentryToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, opts); diff != "" {
+ t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+const (
+ responseBody403 = `
+[
+ {
+ "organization": {
+ "id": "911964",
+ "slug": "wigslap",
+ "status": {
+ "id": "active",
+ "name": "active"
+ },
+ "name": "wigslap"
+ }
+ }
+]
+`
+ responseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
+ responseEmpty = `[]`
+)
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sentrytoken/sentrytoken_test.go b/pkg/detectors/sentrytoken/sentrytoken_test.go
index 5c64febff..8c7806ff4 100644
--- a/pkg/detectors/sentrytoken/sentrytoken_test.go
+++ b/pkg/detectors/sentrytoken/sentrytoken_test.go
@@ -1,229 +1,90 @@
-//go:build detectors
-// +build detectors
-
package sentrytoken
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSentryToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SENTRYTOKEN_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SENTRYTOKEN_INACTIVE")
+var (
+ validPattern = "ad00eba0e2b5b057146e1b2b9373f86dbb0e712d106529111d97cb13f081de20"
+ invalidPattern = "ad00e?a0e2b5b057146e1b2b9373f86dbb0e712d106529111d97cb13f081de20"
+ keyword = "sentrytoken"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSentryToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "found, would be verified but for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found and valid but unexpected api response",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, good key but wrong scope",
- s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sentrytoken",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, account deactivated",
- s: Scanner{client: common.ConstantResponseHttpClient(200, responseAccountDeactivated)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, account deactivated",
- s: Scanner{client: common.ConstantResponseHttpClient(200, responseEmpty)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SentryToken,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, opts); diff != "" {
- t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-const (
- responseBody403 = `
-[
- {
- "organization": {
- "id": "911964",
- "slug": "wigslap",
- "status": {
- "id": "active",
- "name": "active"
- },
- "name": "wigslap"
- }
- }
-]
-`
- responseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
- responseEmpty = `[]`
-)
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/serphouse/serphouse_integration_test.go b/pkg/detectors/serphouse/serphouse_integration_test.go
new file mode 100644
index 000000000..ed6fa8c68
--- /dev/null
+++ b/pkg/detectors/serphouse/serphouse_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package serphouse
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSerphouse_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SERPHOUSE")
+ inactiveSecret := testSecrets.MustGetField("SERPHOUSE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a serphouse secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Serphouse,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a serphouse secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Serphouse,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Serphouse.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Serphouse.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/serphouse/serphouse_test.go b/pkg/detectors/serphouse/serphouse_test.go
index 9182a9e4c..48e1cfd41 100644
--- a/pkg/detectors/serphouse/serphouse_test.go
+++ b/pkg/detectors/serphouse/serphouse_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package serphouse
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSerphouse_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SERPHOUSE")
- inactiveSecret := testSecrets.MustGetField("SERPHOUSE_INACTIVE")
+var (
+ validPattern = "H8VuKAcg3pmwH3NbhOSDJSbVqCxslaOQ7lVqF61be25HEGzCJXzzCGoxw8sE"
+ invalidPattern = "H8VuKAcg3pmwH3Nbh?SDJSbVqCxslaOQ7lVqF61be25HEGzCJXzzCGoxw8sE"
+ keyword = "serphouse"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSerphouse_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a serphouse secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Serphouse,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword serphouse",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a serphouse secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Serphouse,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Serphouse.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Serphouse.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/serpstack/serpstack_integration_test.go b/pkg/detectors/serpstack/serpstack_integration_test.go
new file mode 100644
index 000000000..84cca277c
--- /dev/null
+++ b/pkg/detectors/serpstack/serpstack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package serpstack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSerpStack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SERPSTACK")
+ inactiveSecret := testSecrets.MustGetField("SERPSTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a serpstack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SerpStack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a serpstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SerpStack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SerpStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SerpStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/serpstack/serpstack_test.go b/pkg/detectors/serpstack/serpstack_test.go
index 84cca277c..d2b07e7cb 100644
--- a/pkg/detectors/serpstack/serpstack_test.go
+++ b/pkg/detectors/serpstack/serpstack_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package serpstack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSerpStack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SERPSTACK")
- inactiveSecret := testSecrets.MustGetField("SERPSTACK_INACTIVE")
+var (
+ validPattern = "jeqozza8xili4i96pc1x1ww67lmvrw27"
+ invalidPattern = "jeqozza8xili4i96?c1x1ww67lmvrw27"
+ keyword = "serpstack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSerpStack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a serpstack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SerpStack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword serpstack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a serpstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SerpStack,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SerpStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SerpStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sheety/sheety_integration_test.go b/pkg/detectors/sheety/sheety_integration_test.go
new file mode 100644
index 000000000..3c68d97ed
--- /dev/null
+++ b/pkg/detectors/sheety/sheety_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package sheety
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSheety_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHEETY")
+ id := testSecrets.MustGetField("SHEETY_ID")
+
+ inactiveSecret := testSecrets.MustGetField("SHEETY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sheety,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sheety,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sheety.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sheety.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sheety/sheety_test.go b/pkg/detectors/sheety/sheety_test.go
index d79b45a5f..de4b45f5a 100644
--- a/pkg/detectors/sheety/sheety_test.go
+++ b/pkg/detectors/sheety/sheety_test.go
@@ -1,122 +1,83 @@
-//go:build detectors
-// +build detectors
-
package sheety
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSheety_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHEETY")
- id := testSecrets.MustGetField("SHEETY_ID")
-
- inactiveSecret := testSecrets.MustGetField("SHEETY_INACTIVE")
+var (
+ validKey = "qtb537zc8slokboyekq08tsut9atqa4hnewgjsi2olklupy2z4s3wilqorymakjd"
+ invalidKey = "qtb537zc8slokboyekq08tsut9atqa4h?ewgjsi2olklupy2z4s3wilqorymakjd"
+ validId = "81no0tu89hxci6swi979krdkv87f21z9"
+ invalidId = "81no0tu89hxc?6swi979krdkv87f21z9"
+ keyword = "sheety"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSheety_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sheety,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sheety secret %s within sheetyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sheety,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sheety",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sheety.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Sheety.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sherpadesk/sherpadesk_integration_test.go b/pkg/detectors/sherpadesk/sherpadesk_integration_test.go
new file mode 100644
index 000000000..14145cc98
--- /dev/null
+++ b/pkg/detectors/sherpadesk/sherpadesk_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package sherpadesk
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSherpadesk_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHERPADESK")
+ inactiveSecret := testSecrets.MustGetField("SHERPADESK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sherpadesk,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sherpadesk,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sherpadesk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sherpadesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sherpadesk/sherpadesk_test.go b/pkg/detectors/sherpadesk/sherpadesk_test.go
index c99dd325d..a9f63c4e0 100644
--- a/pkg/detectors/sherpadesk/sherpadesk_test.go
+++ b/pkg/detectors/sherpadesk/sherpadesk_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package sherpadesk
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSherpadesk_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHERPADESK")
- inactiveSecret := testSecrets.MustGetField("SHERPADESK_INACTIVE")
+var (
+ validPattern = "g8wjz4xjtdaoswrmbl2ypcu0neq4cvr4"
+ invalidPattern = "g8wjz4xjtdaoswrm?l2ypcu0neq4cvr4"
+ keyword = "sherpadesk"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSherpadesk_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sherpadesk,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sherpadesk",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sherpadesk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sherpadesk,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sherpadesk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Sherpadesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shipday/shipday_integration_test.go b/pkg/detectors/shipday/shipday_integration_test.go
new file mode 100644
index 000000000..0efc9c528
--- /dev/null
+++ b/pkg/detectors/shipday/shipday_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package shipday
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShipday_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHIPDAY")
+ inactiveSecret := testSecrets.MustGetField("SHIPDAY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shipday secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shipday,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shipday secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shipday,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Shipday.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Shipday.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shipday/shipday_test.go b/pkg/detectors/shipday/shipday_test.go
index c6cc50bf3..2dfc61115 100644
--- a/pkg/detectors/shipday/shipday_test.go
+++ b/pkg/detectors/shipday/shipday_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package shipday
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShipday_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHIPDAY")
- inactiveSecret := testSecrets.MustGetField("SHIPDAY_INACTIVE")
+var (
+ validPattern = "3ajLBL6g6oj2rC9u2vtrkPt8qSTLp07"
+ invalidPattern = "3ajLBL6g6oj2rC9?2vtrkPt8qSTLp07"
+ keyword = "shipday"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShipday_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shipday secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shipday,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shipday",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shipday secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shipday,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Shipday.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Shipday.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shodankey/shodankey_integration_test.go b/pkg/detectors/shodankey/shodankey_integration_test.go
new file mode 100644
index 000000000..d9c730cee
--- /dev/null
+++ b/pkg/detectors/shodankey/shodankey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package shodankey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShodanKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHODANKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SHODANKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shodankey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ShodanKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shodankey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ShodanKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ShodanKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ShodanKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shodankey/shodankey_test.go b/pkg/detectors/shodankey/shodankey_test.go
index d9c730cee..c6ffa6497 100644
--- a/pkg/detectors/shodankey/shodankey_test.go
+++ b/pkg/detectors/shodankey/shodankey_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package shodankey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShodanKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHODANKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SHODANKEY_INACTIVE")
+var (
+ validPattern = "28tEbDdpkFvftp9XkMzAf7O7uVTYlPYD"
+ invalidPattern = "28tEbDdp?Fvftp9XkMzAf7O7uVTYlPYD"
+ keyword = "shodankey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShodanKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shodankey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ShodanKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shodankey",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shodankey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ShodanKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ShodanKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ShodanKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shopify/shopify_integration_test.go b/pkg/detectors/shopify/shopify_integration_test.go
new file mode 100644
index 000000000..0b811f3c9
--- /dev/null
+++ b/pkg/detectors/shopify/shopify_integration_test.go
@@ -0,0 +1,125 @@
+package shopify
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShopify_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET")
+ inactiveSecret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET_INACTIVE")
+ domain := testSecrets.MustGetField("SHOPIFY_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shopify,
+ Redacted: domain,
+ Verified: true,
+ ExtraData: map[string]string{
+ "access_scopes": "read_analytics,read_apps,write_assigned_fulfillment_orders,read_assigned_fulfillment_orders,write_customers,read_customers,write_discovery,read_discovery,write_merchant_managed_fulfillment_orders,read_merchant_managed_fulfillment_orders,write_reports,read_reports,write_cart_transforms,read_cart_transforms",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shopify,
+ Redacted: domain,
+ Verified: false,
+ ExtraData: nil,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Shopify.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ got[i].AnalysisInfo = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shopify/shopify_test.go b/pkg/detectors/shopify/shopify_test.go
index f12ae8976..6936375ff 100644
--- a/pkg/detectors/shopify/shopify_test.go
+++ b/pkg/detectors/shopify/shopify_test.go
@@ -4,122 +4,80 @@ import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShopify_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET")
- inactiveSecret := testSecrets.MustGetField("SHOPIFY_ADMIN_SECRET_INACTIVE")
- domain := testSecrets.MustGetField("SHOPIFY_DOMAIN")
+var (
+ validKey = "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"
+ invalidKey = "shpat_?23eEA6cA03dA2eF5CFe9002Be9F1BD9"
+ validDomain = "phfjg0Fv-Nn52QAMfsb4LfI7gVp7j-J11WykWugPHRhh3n6lGz8eP0.myshopify.com"
+ invalidDomain = "phfjg0Fv-Nn52QAMfsb4LfI7gVp7j-J11W?kWugPHRhh3n6lGz8eP0.myshopify.com"
+ keyword = "shopify"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShopify_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shopify secret %s domain https://%s", secret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shopify,
- Redacted: domain,
- Verified: true,
- ExtraData: map[string]string{
- "access_scopes": "read_analytics,read_apps,write_assigned_fulfillment_orders,read_assigned_fulfillment_orders,write_customers,read_customers,write_discovery,read_discovery,write_merchant_managed_fulfillment_orders,read_merchant_managed_fulfillment_orders,write_reports,read_reports,write_cart_transforms,read_cart_transforms",
- },
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shopify",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),
+ want: []string{validKey + validDomain},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shopify secret %s within (domain https://%s) but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shopify,
- Redacted: domain,
- Verified: false,
- ExtraData: nil,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Shopify.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
- got[i].AnalysisInfo = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shortcut/shortcut_integration_test.go b/pkg/detectors/shortcut/shortcut_integration_test.go
new file mode 100644
index 000000000..c477270db
--- /dev/null
+++ b/pkg/detectors/shortcut/shortcut_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package shortcut
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShortcut_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHORTCUT")
+ inactiveSecret := testSecrets.MustGetField("SHORTCUT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shortcut secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shortcut,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shortcut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shortcut,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Shortcut.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Shortcut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shortcut/shortcut_test.go b/pkg/detectors/shortcut/shortcut_test.go
index c477270db..567872fd5 100644
--- a/pkg/detectors/shortcut/shortcut_test.go
+++ b/pkg/detectors/shortcut/shortcut_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package shortcut
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShortcut_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHORTCUT")
- inactiveSecret := testSecrets.MustGetField("SHORTCUT_INACTIVE")
+var (
+ validPattern = "2e15518b-b3b1-5269-3746-4dd87951caa2"
+ invalidPattern = "2e15518b?b3b1-5269-3746-4dd87951caa2"
+ keyword = "shortcut"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShortcut_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shortcut secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shortcut,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shortcut",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shortcut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shortcut,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Shortcut.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Shortcut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shotstack/shotstack_integration_test.go b/pkg/detectors/shotstack/shotstack_integration_test.go
new file mode 100644
index 000000000..5d4afd1a2
--- /dev/null
+++ b/pkg/detectors/shotstack/shotstack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package shotstack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShotstack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHOTSTACK")
+ inactiveSecret := testSecrets.MustGetField("SHOTSTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shotstack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shotstack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shotstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shotstack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Shotstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Shotstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shotstack/shotstack_test.go b/pkg/detectors/shotstack/shotstack_test.go
index 5a2ec0483..a4060c58a 100644
--- a/pkg/detectors/shotstack/shotstack_test.go
+++ b/pkg/detectors/shotstack/shotstack_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package shotstack
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShotstack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHOTSTACK")
- inactiveSecret := testSecrets.MustGetField("SHOTSTACK_INACTIVE")
+var (
+ validPattern = "3gGdjBjen4jcmMPJa2RJfg5suw47SHihtAznQd4X"
+ invalidPattern = "3gGdjBjen4jcmMPJa2RJ?g5suw47SHihtAznQd4X"
+ keyword = "shotstack"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShotstack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shotstack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shotstack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shotstack",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shotstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shotstack,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Shotstack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Shotstack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shutterstock/shutterstock_integration_test.go b/pkg/detectors/shutterstock/shutterstock_integration_test.go
new file mode 100644
index 000000000..f8df34474
--- /dev/null
+++ b/pkg/detectors/shutterstock/shutterstock_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package shutterstock
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShutterstock_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ key := testSecrets.MustGetField("SHUTTERSTOCK_KEY")
+ inactiveKey := testSecrets.MustGetField("SHUTTERSTOCK_KEY_INACTIVE")
+ secret := testSecrets.MustGetField("SHUTTERSTOCK")
+ inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within", key, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shutterstock,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Shutterstock,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Shutterstock.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Shutterstock.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shutterstock/shutterstock_test.go b/pkg/detectors/shutterstock/shutterstock_test.go
index 00fbe034f..7869ef608 100644
--- a/pkg/detectors/shutterstock/shutterstock_test.go
+++ b/pkg/detectors/shutterstock/shutterstock_test.go
@@ -1,122 +1,83 @@
-//go:build detectors
-// +build detectors
-
package shutterstock
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShutterstock_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- key := testSecrets.MustGetField("SHUTTERSTOCK_KEY")
- inactiveKey := testSecrets.MustGetField("SHUTTERSTOCK_KEY_INACTIVE")
- secret := testSecrets.MustGetField("SHUTTERSTOCK")
- inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCK_INACTIVE")
+var (
+ validKey = "35rqejvHx8E83LC82j65uBHEz8VRlWSw"
+ invalidKey = "35rqejvHx8E83LC8?j65uBHEz8VRlWSw"
+ validSecret = "IJ9u8C6AXnfTwuLd"
+ invalidSecret = "IJ9u8C6A?nfTwuLd"
+ keyword = "shutterstock"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShutterstock_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within", key, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shutterstock,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shutterstock",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
+ want: []string{validSecret},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shutterstock key %s with shutterstock secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Shutterstock,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Shutterstock.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Shutterstock.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth_integration_test.go b/pkg/detectors/shutterstockoauth/shutterstockoauth_integration_test.go
new file mode 100644
index 000000000..cab496900
--- /dev/null
+++ b/pkg/detectors/shutterstockoauth/shutterstockoauth_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package shutterstockoauth
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestShutterstockOAuth_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH")
+ inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ShutterstockOAuth,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ShutterstockOAuth,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ShutterstockOAuth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ShutterstockOAuth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go
index d73d4771f..a79ce7979 100644
--- a/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go
+++ b/pkg/detectors/shutterstockoauth/shutterstockoauth_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package shutterstockoauth
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestShutterstockOAuth_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH")
- inactiveSecret := testSecrets.MustGetField("SHUTTERSTOCKOAUTH_INACTIVE")
+var (
+ validPattern = "v2/G8QwLlOmlwlPzafRJTUNIotsfd3kTTRbtsRldZ3tKdQN98abypvIzPywyqnyAivF9Xev6xhEdJfVIRD8ZVA7YKOjsJ5WNz3QG5XUkw0AiCuBTwDLbeidOMGblASRPwnsGrkfYfN1jX6CPywJsOmfP63zXgBZB6H8iXswjYjj8iMV9NAPQ8OSyDnNuhlX4BGFZ9YdrCJEgzHiavGwteueFTpljPODS8oTXwNtAYV4x06eQ6s4bFAAL2nEzuuSakwnDiJd2wXsXW9cLuMHpawLszYYtDLqApOSyZ33KTfmtrvuWQgTcQv5JfYi93mvCdQS0FJyY5AnKSj9hdvqHf2bsXXK5nEcEN41YcKuU8ADk4uceYwEHWyve4xlIqCpdZRWlif3"
+ invalidPattern = "v1/G8QwLlOmlwlPzafRJTUNIotsfd3kTTRbtsRldZ3tKdQN98abypvIzPywyqnyAivF9Xev6xhEdJfVIRD8ZVA7YKOjsJ5WNz3QG5XUkw0AiCuBTwDLbeidOMGblASRPwnsGrkfYfN1jX6CPywJsOmfP63zXgBZB6H8iXswjYjj8iMV9NAPQ8OSyDnNuhlX4BGFZ9YdrCJEgzHiavGwteueFTpljPODS8oTXwNtAYV4x06eQ6s4bFAAL2nEzuuSakwnDiJd2wXsXW9cLuMHpawLszYYtDLqApOSyZ33KTfmtrvuWQgTcQv5JfYi93mvCdQS0FJyY5AnKSj9hdvqHf2bsXXK5nEcEN41YcKuU8ADk4uceYwEHWyve4xlIqCpdZRWlif3"
+ keyword = "shutterstockoauth"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestShutterstockOAuth_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ShutterstockOAuth,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword shutterstockoauth",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a shutterstockoauth secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ShutterstockOAuth,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ShutterstockOAuth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ShutterstockOAuth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/signable/signable_integration_test.go b/pkg/detectors/signable/signable_integration_test.go
index 42b8d9839..b9d5af5f0 100644
--- a/pkg/detectors/signable/signable_integration_test.go
+++ b/pkg/detectors/signable/signable_integration_test.go
@@ -11,9 +11,8 @@ import (
"github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
diff --git a/pkg/detectors/signalwire/signalwire_integration_test.go b/pkg/detectors/signalwire/signalwire_integration_test.go
new file mode 100644
index 000000000..02f776f04
--- /dev/null
+++ b/pkg/detectors/signalwire/signalwire_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package signalwire
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSignalwire_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("SIGNALWIRE_ID")
+ secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN")
+ url := testSecrets.MustGetField("SIGNALWIRE_URL")
+ inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s", secret, url, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Signalwire,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s but not valid", inactiveSecret, url, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Signalwire,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Signalwire.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Signalwire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.go
index a4c06f265..b4f373916 100644
--- a/pkg/detectors/signalwire/signalwire_test.go
+++ b/pkg/detectors/signalwire/signalwire_test.go
@@ -1,122 +1,85 @@
-//go:build detectors
-// +build detectors
-
package signalwire
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSignalwire_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("SIGNALWIRE_ID")
- secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN")
- url := testSecrets.MustGetField("SIGNALWIRE_URL")
- inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE")
+var (
+ validKey = "rBqctDvrJyZ1MAj2aKHpi6wGcPiCzGtEbCSqtE46RB7yVCywVA"
+ invalidKey = "rBqctDvrJyZ1MAj2aKHpi6wGc?iCzGtEbCSqtE46RB7yVCywVA"
+ validId = "dy0wgsgd-byq5-iqfb-ez1d-dvjepkyub9n9"
+ invalidId = "dy0wgsgd?byq5-iqfb-ez1d-dvjepkyub9n9"
+ validUrl = "09ft358n4gpa1c.signalwire.com"
+ invalidUrl = "09ft358n4gpa1c?signalwire.com"
+ keyword = "signalwire"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSignalwire_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s", secret, url, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Signalwire,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword signalwire",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId, keyword, validUrl),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a signalwire secret %s within signalwire url %s and signalwire id %s but not valid", inactiveSecret, url, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Signalwire,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId, keyword, invalidUrl),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Signalwire.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Signalwire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/signaturit/signaturit_integration_test.go b/pkg/detectors/signaturit/signaturit_integration_test.go
new file mode 100644
index 000000000..b6670237d
--- /dev/null
+++ b/pkg/detectors/signaturit/signaturit_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package signaturit
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSignaturit_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIGNATURIT")
+ inactiveSecret := testSecrets.MustGetField("SIGNATURIT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a signaturit secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Signaturit,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a signaturit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Signaturit,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Signaturit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Signaturit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/signaturit/signaturit_test.go b/pkg/detectors/signaturit/signaturit_test.go
index b706d742e..2175d80af 100644
--- a/pkg/detectors/signaturit/signaturit_test.go
+++ b/pkg/detectors/signaturit/signaturit_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package signaturit
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSignaturit_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIGNATURIT")
- inactiveSecret := testSecrets.MustGetField("SIGNATURIT_INACTIVE")
+var (
+ validPattern = "4AgnWjwDhcgHULFkDkQCY7bra8XfsV9l72JYqsIAFnE9mEo6VusS5yKBJq3OSTEyJfsPoPhorAUc9yUiCxCIxz"
+ invalidPattern = "4AgnWjwDhcgHULFkDkQCY7bra8XfsV?l72JYqsIAFnE9mEo6VusS5yKBJq3OSTEyJfsPoPhorAUc9yUiCxCIxz"
+ keyword = "signaturit"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSignaturit_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a signaturit secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Signaturit,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword signaturit",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a signaturit secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Signaturit,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Signaturit.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Signaturit.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/signupgenius/signupgenius_integration_test.go b/pkg/detectors/signupgenius/signupgenius_integration_test.go
new file mode 100644
index 000000000..e74b007c3
--- /dev/null
+++ b/pkg/detectors/signupgenius/signupgenius_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package signupgenius
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSignupgenius_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIGNUPGENIUS")
+ inactiveSecret := testSecrets.MustGetField("SIGNUPGENIUS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Signupgenius,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Signupgenius,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Signupgenius.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Signupgenius.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/signupgenius/signupgenius_test.go b/pkg/detectors/signupgenius/signupgenius_test.go
index ffb74e529..eacba7d8e 100644
--- a/pkg/detectors/signupgenius/signupgenius_test.go
+++ b/pkg/detectors/signupgenius/signupgenius_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package signupgenius
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSignupgenius_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIGNUPGENIUS")
- inactiveSecret := testSecrets.MustGetField("SIGNUPGENIUS_INACTIVE")
+var (
+ validPattern = "kARx6uGgPqBkho0jzPNfgnAduN3hDjBY"
+ invalidPattern = "kARx6uGgPqBkho0j?PNfgnAduN3hDjBY"
+ keyword = "signupgenius"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSignupgenius_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Signupgenius,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword signupgenius",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a signupgenius secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Signupgenius,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Signupgenius.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Signupgenius.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sigopt/sigopt_integration_test.go b/pkg/detectors/sigopt/sigopt_integration_test.go
new file mode 100644
index 000000000..c25a2cc62
--- /dev/null
+++ b/pkg/detectors/sigopt/sigopt_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package sigopt
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSigopt_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIGOPT")
+ inactiveSecret := testSecrets.MustGetField("SIGOPT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sigopt,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sigopt secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sigopt,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sigopt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sigopt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sigopt/sigopt_test.go b/pkg/detectors/sigopt/sigopt_test.go
index c461d56d0..a0053185f 100644
--- a/pkg/detectors/sigopt/sigopt_test.go
+++ b/pkg/detectors/sigopt/sigopt_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package sigopt
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSigopt_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIGOPT")
- inactiveSecret := testSecrets.MustGetField("SIGOPT_INACTIVE")
+var (
+ validPattern = "GF67UZ5P7UX2A6ELCAE11YDCOYJABCBDYK26O76HRIPU294T"
+ invalidPattern = "GF67UZ5P7UX2A6ELCAE11YDC?YJABCBDYK26O76HRIPU294T"
+ keyword = "sigopt"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSigopt_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sigopt secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sigopt,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sigopt",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sigopt secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sigopt,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sigopt.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Sigopt.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/simfin/simfin_integration_test.go b/pkg/detectors/simfin/simfin_integration_test.go
new file mode 100644
index 000000000..999febb9d
--- /dev/null
+++ b/pkg/detectors/simfin/simfin_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package simfin
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSimFin_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIMFIN")
+ inactiveSecret := testSecrets.MustGetField("SIMFIN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simfin secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SimFin,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simfin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SimFin,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SimFin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SimFin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/simfin/simfin_test.go b/pkg/detectors/simfin/simfin_test.go
index 7ee417e72..66972104b 100644
--- a/pkg/detectors/simfin/simfin_test.go
+++ b/pkg/detectors/simfin/simfin_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package simfin
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSimFin_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIMFIN")
- inactiveSecret := testSecrets.MustGetField("SIMFIN_INACTIVE")
+var (
+ validPattern = "Y28WVaw01VjEEZQ6zn4HvmfpEWT70nPC"
+ invalidPattern = "Y28WVaw01VjEEZQ6?n4HvmfpEWT70nPC"
+ keyword = "simfin"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSimFin_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simfin secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SimFin,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword simfin",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simfin secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SimFin,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SimFin.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SimFin.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/simplesat/simplesat_integration_test.go b/pkg/detectors/simplesat/simplesat_integration_test.go
new file mode 100644
index 000000000..e00c0f31b
--- /dev/null
+++ b/pkg/detectors/simplesat/simplesat_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package simplesat
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSimplesat_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIMPLESAT_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SIMPLESAT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simplesat secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Simplesat,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simplesat secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Simplesat,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Simplesat.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Simplesat.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/simplesat/simplesat_test.go b/pkg/detectors/simplesat/simplesat_test.go
index e00c0f31b..89f4291d9 100644
--- a/pkg/detectors/simplesat/simplesat_test.go
+++ b/pkg/detectors/simplesat/simplesat_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package simplesat
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSimplesat_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIMPLESAT_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SIMPLESAT_INACTIVE")
+var (
+ validPattern = "52dfk83ssx3hnz3ke35jqqucwy5ojdtyd0at5v4c"
+ invalidPattern = "52dfk83ssx3hnz3ke35j?qucwy5ojdtyd0at5v4c"
+ keyword = "simplesat"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSimplesat_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simplesat secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Simplesat,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword simplesat",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simplesat secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Simplesat,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Simplesat.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Simplesat.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/simplynoted/simplynoted_integration_test.go b/pkg/detectors/simplynoted/simplynoted_integration_test.go
new file mode 100644
index 000000000..dff91caa7
--- /dev/null
+++ b/pkg/detectors/simplynoted/simplynoted_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package simplynoted
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSimplyNoted_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIMPLYNOTED")
+ inactiveSecret := testSecrets.MustGetField("SIMPLYNOTED_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SimplyNoted,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SimplyNoted,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SimplyNoted.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SimplyNoted.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/simplynoted/simplynoted_test.go b/pkg/detectors/simplynoted/simplynoted_test.go
index aaf8f3ec8..0e7f3087f 100644
--- a/pkg/detectors/simplynoted/simplynoted_test.go
+++ b/pkg/detectors/simplynoted/simplynoted_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package simplynoted
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSimplyNoted_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIMPLYNOTED")
- inactiveSecret := testSecrets.MustGetField("SIMPLYNOTED_INACTIVE")
+var (
+ validPattern = "NSPwR0>Q,wTEJcGyFF)bvY#FACMwu9DzLFD|bLC@D2d1-zILY&sPo6N_AhOwD(:N0`Td[NL2dZt;>yzn3yN'5x[ia&0v2&M0D_r0!L3#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGaX1WAnoZy2Ny20rnl^f8P/Y.u`jxsOy%2iesTS&u|7SMQ]wJ*f2c?lQ:o4&X(/[y.ZK%2Av100u($ZeTU2N4yCFgKp5PqicqSkgjIla31uGS0OvpmpSiy@rFvthHA,k&)uRAM6$>#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGJcGyFF)bvY#FACMwu9DzL"
+ invalidPattern = "N PwR0>Q,wTEJcGyFF)bvY#FACMwu9DzLFD|bLC@D2d1-zILY&sPo6N_AhOwD(:N0`Td[NL2dZt;>yzn3yN'5x[ia&0v2&M0D_r0!L3#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGaX1WAnoZy2Ny20rnl^f8P/Y.u`jxsOy%2iesTS&u|7SMQ]wJ*f2c?lQ:o4&X(/[y.ZK%2Av100u($ZeTU2N4yCFgKp5PqicqSkgjIla31uGS0OvpmpSiy@rFvthHA,k&)uRAM6$>#dt:Mx^IdcL{xSBIlyQ4jd*=4U2=gdr%2ZDvQqxGJcGyFF)bvY#FACMwu9DzL"
+ keyword = "simplynoted"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSimplyNoted_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SimplyNoted,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword simplynoted",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simplynoted secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SimplyNoted,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SimplyNoted.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SimplyNoted.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/simvoly/simvoly_integration_test.go b/pkg/detectors/simvoly/simvoly_integration_test.go
new file mode 100644
index 000000000..af28be91a
--- /dev/null
+++ b/pkg/detectors/simvoly/simvoly_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package simvoly
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSimvoly_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIMVOLY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SIMVOLY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simvoly secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Simvoly,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a simvoly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Simvoly,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Simvoly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Simvoly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/simvoly/simvoly_test.go b/pkg/detectors/simvoly/simvoly_test.go
index 99585908c..fc10692dc 100644
--- a/pkg/detectors/simvoly/simvoly_test.go
+++ b/pkg/detectors/simvoly/simvoly_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package simvoly
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSimvoly_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIMVOLY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SIMVOLY_INACTIVE")
+var (
+ validPattern = "zu0rpr33yvk0c18vs5wzpx2nq9ww1bh2b"
+ invalidPattern = "zu0rpr33yvk0c18v?5wzpx2nq9ww1bh2b"
+ keyword = "simvoly"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSimvoly_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simvoly secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Simvoly,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword simvoly",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a simvoly secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Simvoly,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Simvoly.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Simvoly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sinchmessage/sinchmessage_integration_test.go b/pkg/detectors/sinchmessage/sinchmessage_integration_test.go
new file mode 100644
index 000000000..a40178820
--- /dev/null
+++ b/pkg/detectors/sinchmessage/sinchmessage_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package sinchmessage
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSinchMessage_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SINCHMESSAGE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SINCHMESSAGE_INACTIVE")
+ id := testSecrets.MustGetField("SINCHMESSAGE_PLANID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within sinchid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SinchMessage,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within but sinchid %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SinchMessage,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SinchMessage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SinchMessage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sinchmessage/sinchmessage_test.go b/pkg/detectors/sinchmessage/sinchmessage_test.go
index 022f04419..53035bb0a 100644
--- a/pkg/detectors/sinchmessage/sinchmessage_test.go
+++ b/pkg/detectors/sinchmessage/sinchmessage_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package sinchmessage
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSinchMessage_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SINCHMESSAGE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SINCHMESSAGE_INACTIVE")
- id := testSecrets.MustGetField("SINCHMESSAGE_PLANID")
+var (
+ validKey = "k255nuzmx8fzlkjfjz5unlrdbd658bon"
+ invalidKey = "k255nuzmx8fzlk?fjz5unlrdbd658bon"
+ validId = "y9yvuc6hl2up5qyohet77j23ntll7pr7"
+ invalidId = "y9yvuc6?l2up5qyohet77j23ntll7pr7"
+ keyword = "sinchmessage"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSinchMessage_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within sinchid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SinchMessage,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sinchmessage",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey, validId, validKey, validId},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sinchmessage secret %s within but sinchid %s not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SinchMessage,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SinchMessage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SinchMessage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sirv/sirv_integration_test.go b/pkg/detectors/sirv/sirv_integration_test.go
new file mode 100644
index 000000000..1905f6a4f
--- /dev/null
+++ b/pkg/detectors/sirv/sirv_integration_test.go
@@ -0,0 +1,118 @@
+//go:build detectors
+// +build detectors
+
+package sirv
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSirv_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SIRV")
+ id := testSecrets.MustGetField("SIRV_ID")
+ inactiveSecret := testSecrets.MustGetField("SIRV_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sirv,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sirv,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sirv.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sirv.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sirv/sirv_test.go b/pkg/detectors/sirv/sirv_test.go
index 1905f6a4f..3b17bc099 100644
--- a/pkg/detectors/sirv/sirv_test.go
+++ b/pkg/detectors/sirv/sirv_test.go
@@ -1,117 +1,82 @@
-//go:build detectors
-// +build detectors
-
package sirv
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSirv_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SIRV")
- id := testSecrets.MustGetField("SIRV_ID")
- inactiveSecret := testSecrets.MustGetField("SIRV_INACTIVE")
+var (
+ validKey = "Ivus86]AJDk:rghv;8,o00SVP0efds5BBh453jew~mnC$K8elZI-OGU^v0AA#`>vek1cKlkipm]!n/f1xTDlUn1D"
+ invalidKey = "?vus86]AJDk:rghv;8,o00SVP0efds5BBh453jew~mnC$K8elZI-OGU^v0AA#`>vek1cKlkipm]!n/f1xTDlUn1D"
+ validId = "n64cjK3Dg9Xyr2icHYypLwXl6V"
+ invalidId = "n64cjK3Dg9Xyr?icHYypLwXl6V"
+ keyword = "sirv"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSirv_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sirv,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sirv",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sirv secret %s within sirv %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sirv,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sirv.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Sirv.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/siteleaf/siteleaf_integration_test.go b/pkg/detectors/siteleaf/siteleaf_integration_test.go
new file mode 100644
index 000000000..e55d124a5
--- /dev/null
+++ b/pkg/detectors/siteleaf/siteleaf_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package siteleaf
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSiteleaf_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SITELEAF_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SITELEAF_INACTIVE")
+ id := testSecrets.MustGetField("SITELEAF_USERID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Siteleaf,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Siteleaf,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Siteleaf.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Siteleaf.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/siteleaf/siteleaf_test.go b/pkg/detectors/siteleaf/siteleaf_test.go
index 64da43b06..1f1f716b9 100644
--- a/pkg/detectors/siteleaf/siteleaf_test.go
+++ b/pkg/detectors/siteleaf/siteleaf_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package siteleaf
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSiteleaf_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SITELEAF_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SITELEAF_INACTIVE")
- id := testSecrets.MustGetField("SITELEAF_USERID")
+var (
+ validKey = "lklep3linr2itd42nkncw9b8s9g1yrgb"
+ invalidKey = "lklep3li?r2itd42nkncw9b8s9g1yrgb"
+ validId = "zoc07j72vs6q1lxs6j5vkzz7mefhiii6"
+ invalidId = "zoc07j72vs?q1lxs6j5vkzz7mefhiii6"
+ keyword = "siteleaf"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSiteleaf_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Siteleaf,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword siteleaf",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey, validId, validKey, validId},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a siteleafkey secret %s within siteleafid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Siteleaf,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Siteleaf.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Siteleaf.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/skrappio/skrappio_integration_test.go b/pkg/detectors/skrappio/skrappio_integration_test.go
new file mode 100644
index 000000000..709f782f7
--- /dev/null
+++ b/pkg/detectors/skrappio/skrappio_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package skrappio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSkrapio_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SKRAPIO_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SKRAPIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a skrapp secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Skrappio,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a skrapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Skrappio,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Skrapio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Skrapio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/skrappio/skrappio_test.go b/pkg/detectors/skrappio/skrappio_test.go
index a1520533d..ee8222cfd 100644
--- a/pkg/detectors/skrappio/skrappio_test.go
+++ b/pkg/detectors/skrappio/skrappio_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package skrappio
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSkrapio_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SKRAPIO_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SKRAPIO_INACTIVE")
+var (
+ validPattern = "rAheANMWIEHz9GfmtocXjvG8ESfdZvgSMTbDBIR7SA"
+ invalidPattern = "rAheANMWIEHz9GfmtocXj?G8ESfdZvgSMTbDBIR7SA"
+ keyword = "skrappio"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSkrapio_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a skrapp secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Skrappio,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword skrappio",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a skrapp secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Skrappio,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Skrapio.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Skrapio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/skybiometry/skybiometry_integration_test.go b/pkg/detectors/skybiometry/skybiometry_integration_test.go
new file mode 100644
index 000000000..f7d96e29b
--- /dev/null
+++ b/pkg/detectors/skybiometry/skybiometry_integration_test.go
@@ -0,0 +1,130 @@
+//go:build detectors
+// +build detectors
+
+package skybiometry
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSkyBiometry_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ key := testSecrets.MustGetField("SKYBIOMETRY_KEY")
+ inactiveKey := testSecrets.MustGetField("SKYBIOMETRY_KEY_INACTIVE")
+ secret := testSecrets.MustGetField("SKYBIOMETRY")
+ inactiveSecret := testSecrets.MustGetField("SKYBIOMETRY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within", key, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SkyBiometry,
+ Verified: true,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_SkyBiometry,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SkyBiometry,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_SkyBiometry,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SkyBiometry.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SkyBiometry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/skybiometry/skybiometry_test.go b/pkg/detectors/skybiometry/skybiometry_test.go
index f7d96e29b..f052f95e0 100644
--- a/pkg/detectors/skybiometry/skybiometry_test.go
+++ b/pkg/detectors/skybiometry/skybiometry_test.go
@@ -1,130 +1,83 @@
-//go:build detectors
-// +build detectors
-
package skybiometry
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSkyBiometry_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- key := testSecrets.MustGetField("SKYBIOMETRY_KEY")
- inactiveKey := testSecrets.MustGetField("SKYBIOMETRY_KEY_INACTIVE")
- secret := testSecrets.MustGetField("SKYBIOMETRY")
- inactiveSecret := testSecrets.MustGetField("SKYBIOMETRY_INACTIVE")
+var (
+ validKey = "qdmxigxqpkpo8i38zxfuxct3d2"
+ invalidKey = "qdmx?gxqpkpo8i38zxfuxct3d2"
+ validSecret = "ko86ttplsbrao397eub88pqniu"
+ invalidSecret = "ko86ttplsbrao?97eub88pqniu"
+ keyword = "skybiometry"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSkyBiometry_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within", key, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SkyBiometry,
- Verified: true,
- },
- {
- DetectorType: detectorspb.DetectorType_SkyBiometry,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a skybiometry key %s with skybiometry secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SkyBiometry,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_SkyBiometry,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword skybiometry",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
+ want: []string{validKey, validSecret},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SkyBiometry.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SkyBiometry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/slack/slack_integration_test.go b/pkg/detectors/slack/slack_integration_test.go
new file mode 100644
index 000000000..19d696753
--- /dev/null
+++ b/pkg/detectors/slack/slack_integration_test.go
@@ -0,0 +1,189 @@
+//go:build detectors
+// +build detectors
+
+package slack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSlack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SLACK")
+ secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ wantResults []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+ verify: true,
+ },
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: true,
+ ExtraData: map[string]string{
+ "name": "marge.haskell.bridge",
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
+ "team": "ct.org",
+ "token_type": "Slack User Token",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found but unverified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
+ verify: true,
+ },
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: false,
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
+ "token_type": "Slack User Token",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "account_inactive",
+ s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "account_inactive"}`)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+ verify: true,
+ },
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: false,
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
+ "token_type": "Slack User Token",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+ verify: true,
+ },
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: false,
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
+ "token_type": "Slack User Token",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "unexpected auth response",
+ s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
+ verify: true,
+ },
+ wantResults: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Slack,
+ Verified: false,
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
+ "token_type": "Slack User Token",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ got[i].AnalysisInfo = nil
+
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
+ t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/slack/slack_test.go b/pkg/detectors/slack/slack_test.go
index 19d696753..49f8e3c2e 100644
--- a/pkg/detectors/slack/slack_test.go
+++ b/pkg/detectors/slack/slack_test.go
@@ -1,188 +1,86 @@
-//go:build detectors
-// +build detectors
-
package slack
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSlack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SLACK")
- secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validBotToken = "xoxb-65677559833-9613778673399u"
+ invalidBotToken = "xoxb-65677559833?9613778673399u"
+ validUserToken = "xoxp-24232636415-0285024315463c"
+ invalidUserToken = "xoxp-24232636415?0285024315463c"
+ validWorkspaceAccessToken = "xoxa-08532509747-07570405353c"
+ invalidWorkspaceAccessToken = "xoxa-08532509747?07570405353c"
+ validWorkspaceRefreshToken = "xoxr-833485595373-24619897332l"
+ invalidWorkspaceRefreshToken = "xoxr-833485595373?24619897332l"
+ keyword = "slack"
+)
+func TestSlack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- wantResults []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
- verify: true,
- },
- wantResults: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Slack,
- Verified: true,
- ExtraData: map[string]string{
- "name": "marge.haskell.bridge",
- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
- "team": "ct.org",
- "token_type": "Slack User Token",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "found but unverified",
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
- verify: true,
- },
- wantResults: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Slack,
- Verified: false,
- ExtraData: map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
- "token_type": "Slack User Token",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "account_inactive",
- s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "account_inactive"}`)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
- verify: true,
- },
- wantResults: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Slack,
- Verified: false,
- ExtraData: map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
- "token_type": "Slack User Token",
- },
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
- verify: true,
- },
- wantResults: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Slack,
- Verified: false,
- ExtraData: map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
- "token_type": "Slack User Token",
- },
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "valid pattern - with keyword slack",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validBotToken, keyword, validUserToken, keyword, validWorkspaceAccessToken, keyword, validWorkspaceRefreshToken),
+ want: []string{validBotToken, validUserToken, validWorkspaceAccessToken, validWorkspaceRefreshToken},
},
{
- name: "unexpected auth response",
- s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
- verify: true,
- },
- wantResults: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Slack,
- Verified: false,
- ExtraData: map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/slack/",
- "token_type": "Slack User Token",
- },
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidBotToken, keyword, invalidUserToken, keyword, invalidWorkspaceAccessToken, keyword, invalidWorkspaceRefreshToken),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
- got[i].AnalysisInfo = nil
+ return
+ }
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
- t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/smartsheets/smartsheets_integration_test.go b/pkg/detectors/smartsheets/smartsheets_integration_test.go
new file mode 100644
index 000000000..386ec16e9
--- /dev/null
+++ b/pkg/detectors/smartsheets/smartsheets_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package smartsheets
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSmartsheets_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SMARTSHEETS_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SMARTSHEETS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Smartsheets,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Smartsheets,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Smartsheets.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Smartsheets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/smartsheets/smartsheets_test.go b/pkg/detectors/smartsheets/smartsheets_test.go
index 81facebc8..a388b7304 100644
--- a/pkg/detectors/smartsheets/smartsheets_test.go
+++ b/pkg/detectors/smartsheets/smartsheets_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package smartsheets
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSmartsheets_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SMARTSHEETS_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SMARTSHEETS_INACTIVE")
+var (
+ validPattern = "MVE7zmdxouvunYkowLzaudyX7tvMpkqJ3q52C"
+ invalidPattern = "MVE7?mdxouvunYkowLzaudyX7tvMpkqJ3q52C"
+ keyword = "smartsheets"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSmartsheets_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Smartsheets,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword smartsheets",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a smartsheets secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Smartsheets,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Smartsheets.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Smartsheets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/smartystreets/smartystreets_integration_test.go b/pkg/detectors/smartystreets/smartystreets_integration_test.go
new file mode 100644
index 000000000..a4a34abbe
--- /dev/null
+++ b/pkg/detectors/smartystreets/smartystreets_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package smartystreets
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSmartyStreets_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SMARTYSTREETS")
+ id := testSecrets.MustGetField("SMARTYSTREETS_ID")
+ inactiveSecret := testSecrets.MustGetField("SMARTYSTREETS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SmartyStreets,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SmartyStreets,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SmartyStreets.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SmartyStreets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/smartystreets/smartystreets_test.go b/pkg/detectors/smartystreets/smartystreets_test.go
index b293f5914..cd04045f5 100644
--- a/pkg/detectors/smartystreets/smartystreets_test.go
+++ b/pkg/detectors/smartystreets/smartystreets_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package smartystreets
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSmartyStreets_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SMARTYSTREETS")
- id := testSecrets.MustGetField("SMARTYSTREETS_ID")
- inactiveSecret := testSecrets.MustGetField("SMARTYSTREETS_INACTIVE")
+var (
+ validKey = "UzwkWoJlOqT5q9pXbX2n"
+ invalidKey = "UzwkWoJlOq?5q9pXbX2n"
+ validId = "pers5g2aa5icwflc1cmqjrs3ij8iqjmsg1x3"
+ invalidId = "pers5g2aa5icwflc1c?qjrs3ij8iqjmsg1x3"
+ keyword = "smartystreets"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSmartyStreets_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SmartyStreets,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword smartystreets",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a smartystreets secret %s within smartystreets %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SmartyStreets,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SmartyStreets.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SmartyStreets.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/smooch/smooch_integration_test.go b/pkg/detectors/smooch/smooch_integration_test.go
new file mode 100644
index 000000000..b982b5213
--- /dev/null
+++ b/pkg/detectors/smooch/smooch_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package smooch
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSmooch_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SMOOCH_SECRET")
+ id := testSecrets.MustGetField("SMOOCH_ID")
+ inactiveID := testSecrets.MustGetField("SMOOCH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Smooch,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Smooch,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Smooch.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Smooch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/smooch/smooch_test.go b/pkg/detectors/smooch/smooch_test.go
index 7c5030306..a68cd36f5 100644
--- a/pkg/detectors/smooch/smooch_test.go
+++ b/pkg/detectors/smooch/smooch_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package smooch
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSmooch_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SMOOCH_SECRET")
- id := testSecrets.MustGetField("SMOOCH_ID")
- inactiveID := testSecrets.MustGetField("SMOOCH_INACTIVE")
+var (
+ validKey = "act_svf99w1upfk2syajjx6wxxg9"
+ invalidKey = "act_svf99w1upf?2syajjx6wxxg9"
+ validSecret = "w2XXiJgDfMJnkKQMPKEF4AzeHnQseEcIsK0C7Gm0icA5PtRM4GIZl82B84n01RJMmkz3aGHtpP_44UXXKhW76L"
+ invalidSecret = "w2XXiJgDfMJnkKQMPKEF4AzeHnQseEcIsK0C7Gm0icA?PtRM4GIZl82B84n01RJMmkz3aGHtpP_44UXXKhW76L"
+ keyword = "smooch"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSmooch_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Smooch,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword smooch",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSecret),
+ want: []string{validKey + validSecret},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a smooch secret %s within smooch id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Smooch,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSecret),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Smooch.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Smooch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/snipcart/snipcart_integration_test.go b/pkg/detectors/snipcart/snipcart_integration_test.go
new file mode 100644
index 000000000..35be88a33
--- /dev/null
+++ b/pkg/detectors/snipcart/snipcart_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package snipcart
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSnipcart_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SNIPCART")
+ inactiveSecret := testSecrets.MustGetField("SNIPCART_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a snipcart secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Snipcart,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a snipcart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Snipcart,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Snipcart.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Snipcart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/snipcart/snipcart_test.go b/pkg/detectors/snipcart/snipcart_test.go
index be3328a7f..a8a754fb8 100644
--- a/pkg/detectors/snipcart/snipcart_test.go
+++ b/pkg/detectors/snipcart/snipcart_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package snipcart
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSnipcart_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SNIPCART")
- inactiveSecret := testSecrets.MustGetField("SNIPCART_INACTIVE")
+var (
+ validPattern = "zc62fqbZm4zyVsdPFAcaRhFnSFpO12Lpmas3PU9IK5CJEklhJDeybwRr8c7ibBG3fL3iWoXTHbb"
+ invalidPattern = "zc62fqbZm4zyVsdPFAcaRhFnSFpO12Lpmas3P?9IK5CJEklhJDeybwRr8c7ibBG3fL3iWoXTHbb"
+ keyword = "snipcart"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSnipcart_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a snipcart secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Snipcart,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword snipcart",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a snipcart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Snipcart,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Snipcart.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Snipcart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/snowflake/snowflake_integration_test.go b/pkg/detectors/snowflake/snowflake_integration_test.go
new file mode 100644
index 000000000..d7d7cdbe9
--- /dev/null
+++ b/pkg/detectors/snowflake/snowflake_integration_test.go
@@ -0,0 +1,175 @@
+//go:build detectors
+// +build detectors
+
+package snowflake
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSnowflake_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+
+ accountIdentifier := testSecrets.MustGetField("SNOWFLAKE_ACCOUNT")
+ username := testSecrets.MustGetField("SNOWFLAKE_USERNAME")
+ password := testSecrets.MustGetField("SNOWFLAKE_PASS")
+ inactivePassword := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE")
+
+ // Create a context with a past deadline to simulate DeadlineExceeded error
+ pastTime := time.Now().Add(-time.Second) // Set the deadline in the past
+ errorCtx, cancel := context.WithDeadline(context.Background(), pastTime)
+ defer cancel()
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Snowflake,
+ Verified: true,
+ ExtraData: map[string]string{
+ "account": accountIdentifier,
+ "username": username,
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, inactivePassword)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Snowflake,
+ Verified: false,
+ ExtraData: map[string]string{
+ "account": accountIdentifier,
+ "username": username,
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, indeterminate error (timeout)",
+ s: Scanner{},
+ args: args{
+ ctx: errorCtx,
+ data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Snowflake,
+ ExtraData: map[string]string{
+ "account": accountIdentifier,
+ "username": username,
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Snowflake.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ keysToCopy := []string{"account", "username"}
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+
+ got[i].ExtraData = newMap(got[i].ExtraData, keysToCopy)
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Snowflake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func newMap(extraMap map[string]string, keysToCopy []string) map[string]string {
+ newExtraDataMap := make(map[string]string)
+ for _, key := range keysToCopy {
+ if value, ok := extraMap[key]; ok {
+ newExtraDataMap[key] = value
+ }
+ }
+ return newExtraDataMap
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/snowflake/snowflake_test.go b/pkg/detectors/snowflake/snowflake_test.go
index f06a872fd..45acafa11 100644
--- a/pkg/detectors/snowflake/snowflake_test.go
+++ b/pkg/detectors/snowflake/snowflake_test.go
@@ -1,21 +1,14 @@
-//go:build detectors
-// +build detectors
-
package snowflake
import (
"context"
"fmt"
"testing"
- "time"
"github.com/brianvoe/gofakeit/v7"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestSnowflake_Pattern(t *testing.T) {
@@ -104,160 +97,3 @@ func TestSnowflake_Pattern(t *testing.T) {
})
}
}
-
-func TestSnowflake_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
-
- accountIdentifier := testSecrets.MustGetField("SNOWFLAKE_ACCOUNT")
- username := testSecrets.MustGetField("SNOWFLAKE_USERNAME")
- password := testSecrets.MustGetField("SNOWFLAKE_PASS")
- inactivePassword := testSecrets.MustGetField("SNOWFLAKE_PASS_INACTIVE")
-
- // Create a context with a past deadline to simulate DeadlineExceeded error
- pastTime := time.Now().Add(-time.Second) // Set the deadline in the past
- errorCtx, cancel := context.WithDeadline(context.Background(), pastTime)
- defer cancel()
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Snowflake,
- Verified: true,
- ExtraData: map[string]string{
- "account": accountIdentifier,
- "username": username,
- },
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, inactivePassword)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Snowflake,
- Verified: false,
- ExtraData: map[string]string{
- "account": accountIdentifier,
- "username": username,
- },
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, indeterminate error (timeout)",
- s: Scanner{},
- args: args{
- ctx: errorCtx,
- data: []byte(fmt.Sprintf("snowflake: \n account=%s \n username=%s \n password=%s \n database=SNOWFLAKE", accountIdentifier, username, password)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Snowflake,
- ExtraData: map[string]string{
- "account": accountIdentifier,
- "username": username,
- },
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Snowflake.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- keysToCopy := []string{"account", "username"}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
- }
-
- got[i].ExtraData = newMap(got[i].ExtraData, keysToCopy)
- }
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Snowflake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func newMap(extraMap map[string]string, keysToCopy []string) map[string]string {
- newExtraDataMap := make(map[string]string)
- for _, key := range keysToCopy {
- if value, ok := extraMap[key]; ok {
- newExtraDataMap[key] = value
- }
- }
- return newExtraDataMap
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
- }
- })
- }
-}
diff --git a/pkg/detectors/sonarcloud/sonarcloud_integration_test.go b/pkg/detectors/sonarcloud/sonarcloud_integration_test.go
new file mode 100644
index 000000000..fb9554885
--- /dev/null
+++ b/pkg/detectors/sonarcloud/sonarcloud_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package sonarcloud
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSonarCloud_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SONARCLOUD")
+ inactiveSecret := testSecrets.MustGetField("SONARCLOUD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SonarCloud,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SonarCloud,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SonarCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SonarCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sonarcloud/sonarcloud_test.go b/pkg/detectors/sonarcloud/sonarcloud_test.go
index 2d1720b2c..142b34352 100644
--- a/pkg/detectors/sonarcloud/sonarcloud_test.go
+++ b/pkg/detectors/sonarcloud/sonarcloud_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package sonarcloud
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSonarCloud_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SONARCLOUD")
- inactiveSecret := testSecrets.MustGetField("SONARCLOUD_INACTIVE")
+var (
+ validPattern = "hxrxxgxtcjxj7ta3dn33c5r5i2h0i6cqjv9kwkye"
+ invalidPattern = "hxrxxgxt?jxj7ta3dn33c5r5i2h0i6cqjv9kwkye"
+ keyword = "sonarcloud"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSonarCloud_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SonarCloud,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sonarcloud",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sonarcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SonarCloud,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SonarCloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SonarCloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sourcegraph/sourcegraph_integration_test.go b/pkg/detectors/sourcegraph/sourcegraph_integration_test.go
new file mode 100644
index 000000000..fb695ca4e
--- /dev/null
+++ b/pkg/detectors/sourcegraph/sourcegraph_integration_test.go
@@ -0,0 +1,253 @@
+//go:build detectors
+// +build detectors
+
+package sourcegraph
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSourcegraph_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+
+ secretV1 := testSecrets.MustGetField("SOURCEGRAPH_V1")
+ secretV2 := testSecrets.MustGetField("SOURCEGRAPH_V2")
+ secretV3 := testSecrets.MustGetField("SOURCEGRAPH_V3")
+
+ inactiveSecretV1 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V1")
+ inactiveSecretV2 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V2")
+ inactiveSecretV3 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V3")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified v1",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, verified v1",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, verified v2",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV2)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, verified v3",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV3)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified v1",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV1)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified v2",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV2)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified v3",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV3)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sourcegraph,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ got[i].AnalysisInfo = nil
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError", "ExtraData")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sourcegraph/sourcegraph_test.go b/pkg/detectors/sourcegraph/sourcegraph_test.go
index c15c09527..8e7bcf84b 100644
--- a/pkg/detectors/sourcegraph/sourcegraph_test.go
+++ b/pkg/detectors/sourcegraph/sourcegraph_test.go
@@ -1,253 +1,80 @@
-//go:build detectors
-// +build detectors
-
package sourcegraph
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSourcegraph_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
-
- secretV1 := testSecrets.MustGetField("SOURCEGRAPH_V1")
- secretV2 := testSecrets.MustGetField("SOURCEGRAPH_V2")
- secretV3 := testSecrets.MustGetField("SOURCEGRAPH_V3")
-
- inactiveSecretV1 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V1")
- inactiveSecretV2 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V2")
- inactiveSecretV3 := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE_V3")
+var (
+ validPattern = "sgp_4a2aD05B07721CE0_EEBdEacB356CdC6Dc8CD317A4276e2cEE9cb118a"
+ invalidPattern = "sgp_4a2aD0?B07721CE0_EEBdEacB356CdC6Dc8CD317A4276e2cEE9cb118a"
+ keyword = "sourcegraph"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSourcegraph_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified v1",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, verified v1",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword sourcegraph",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, verified v2",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV2)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, verified v3",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV3)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified v1",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV1)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified v2",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV2)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified v3",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecretV3)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secretV1)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sourcegraph,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
- got[i].AnalysisInfo = nil
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError", "ExtraData")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/sourcegraphcody/sourcegraphcody_integration_test.go b/pkg/detectors/sourcegraphcody/sourcegraphcody_integration_test.go
new file mode 100644
index 000000000..824beac3c
--- /dev/null
+++ b/pkg/detectors/sourcegraphcody/sourcegraphcody_integration_test.go
@@ -0,0 +1,161 @@
+//go:build detectors
+// +build detectors
+
+package sourcegraphcody
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSourcegraphcody_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SOURCEGRAPHCODY")
+ inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SourcegraphCody,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SourcegraphCody,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SourcegraphCody,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SourcegraphCody,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go b/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go
index 2445ebfa2..cac9cd42f 100644
--- a/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go
+++ b/pkg/detectors/sourcegraphcody/sourcegraphcody_test.go
@@ -1,161 +1,80 @@
-//go:build detectors
-// +build detectors
-
package sourcegraphcody
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSourcegraphcody_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SOURCEGRAPHCODY")
- inactiveSecret := testSecrets.MustGetField("SOURCEGRAPHCODY_INACTIVE")
+var (
+ validPattern = "slk_3c6669a93b14fbd6061e00fb40876b327826df4fe393e760d3109de08165e793"
+ invalidPattern = "slk_?c6669a93b14fbd6061e00fb40876b327826df4fe393e760d3109de08165e793"
+ keyword = "sourcegraphcody"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSourcegraphcody_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SourcegraphCody,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SourcegraphCody,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword sourcegraphcody",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SourcegraphCody,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sourcegraphcody secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SourcegraphCody,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sourcegraphcody.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Sourcegraphcody.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/sparkpost/sparkpost_integration_test.go b/pkg/detectors/sparkpost/sparkpost_integration_test.go
new file mode 100644
index 000000000..6711fe639
--- /dev/null
+++ b/pkg/detectors/sparkpost/sparkpost_integration_test.go
@@ -0,0 +1,123 @@
+//go:build detectors
+// +build detectors
+
+package sparkpost
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSparkpost_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SPARKPOST_API_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SPARKPOST_API_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sparkpost,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sparkpost,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sparkpost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+
+ t.Errorf("Sparkpost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sparkpost/sparkpost_test.go b/pkg/detectors/sparkpost/sparkpost_test.go
index f82992dda..468120100 100644
--- a/pkg/detectors/sparkpost/sparkpost_test.go
+++ b/pkg/detectors/sparkpost/sparkpost_test.go
@@ -1,123 +1,81 @@
-//go:build detectors
-// +build detectors
-
package sparkpost
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSparkpost_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SPARKPOST_API_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SPARKPOST_API_INACTIVE")
+var (
+ validPattern = "VlsweyCt4GRglwZ8G3xVo5lY8Z4Wo9cLxyHFWh9V"
+ invalidPattern = "VlsweyCt4GRglwZ8G3xV?5lY8Z4Wo9cLxyHFWh9V"
+ keyword = "sparkpost"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSparkpost_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sparkpost,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sparkpost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sparkpost,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sparkpost",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sparkpost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
- t.Errorf("Sparkpost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/speechtextai/speechtextai_integration_test.go b/pkg/detectors/speechtextai/speechtextai_integration_test.go
new file mode 100644
index 000000000..f3184a5fd
--- /dev/null
+++ b/pkg/detectors/speechtextai/speechtextai_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package speechtextai
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSpeechTextAI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SPEECHTEXTAI")
+ inactiveSecret := testSecrets.MustGetField("SPEECHTEXTAI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a speechtext secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SpeechTextAI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a speechtext secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SpeechTextAI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SpeechTextAI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SpeechTextAI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/speechtextai/speechtextai_test.go b/pkg/detectors/speechtextai/speechtextai_test.go
index f759c6ce9..6c1ef9183 100644
--- a/pkg/detectors/speechtextai/speechtextai_test.go
+++ b/pkg/detectors/speechtextai/speechtextai_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package speechtextai
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSpeechTextAI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SPEECHTEXTAI")
- inactiveSecret := testSecrets.MustGetField("SPEECHTEXTAI_INACTIVE")
+var (
+ validPattern = "51bb70d3ab4f9e0ecd17ec0a6f8ed289"
+ invalidPattern = "G1bb70d3ab4f9e0ecd17ec0a6f8ed289"
+ keyword = "speechtextai"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSpeechTextAI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a speechtext secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SpeechTextAI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword speechtextai",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a speechtext secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SpeechTextAI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SpeechTextAI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SpeechTextAI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_integration_test.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_integration_test.go
new file mode 100644
index 000000000..49de9e00a
--- /dev/null
+++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package splunkobservabilitytoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSplunkObservabilityToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SPLUNK_ACCESS_TOKEN") // TODO: rename
+ inactiveSecret := testSecrets.MustGetField("SPLUNK_ACCESS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a splunk secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a splunk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SplunkObservabilityToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SplunkObservabilityToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go
index 6d172f8fa..9cf7298d1 100644
--- a/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go
+++ b/pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package splunkobservabilitytoken
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSplunkObservabilityToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SPLUNK_ACCESS_TOKEN") // TODO: rename
- inactiveSecret := testSecrets.MustGetField("SPLUNK_ACCESS_INACTIVE")
+var (
+ validPattern = "OlKU6qOr3U4AlCziwvFCY2"
+ invalidPattern = "OlKU6qOr3U4?lCziwvFCY2"
+ keyword = "splunkobservabilitytoken"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSplunkObservabilityToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a splunk secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword splunkobservabilitytoken",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a splunk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SplunkOberservabilityToken,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SplunkObservabilityToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SplunkObservabilityToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/spoonacular/spoonacular_integration_test.go b/pkg/detectors/spoonacular/spoonacular_integration_test.go
new file mode 100644
index 000000000..91ba6069b
--- /dev/null
+++ b/pkg/detectors/spoonacular/spoonacular_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package spoonacular
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSpoonacular_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SPOONACULAR")
+ inactiveSecret := testSecrets.MustGetField("SPOONACULAR_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Spoonacular,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Spoonacular,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Spoonacular.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Spoonacular.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/spoonacular/spoonacular_test.go b/pkg/detectors/spoonacular/spoonacular_test.go
index f4a786aca..d8d5e323b 100644
--- a/pkg/detectors/spoonacular/spoonacular_test.go
+++ b/pkg/detectors/spoonacular/spoonacular_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package spoonacular
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSpoonacular_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SPOONACULAR")
- inactiveSecret := testSecrets.MustGetField("SPOONACULAR_INACTIVE")
+var (
+ validPattern = "xyclb6zdiq9hyexe74ah4lvp3ea02sbv"
+ invalidPattern = "xyclb6zdiq9hyexe?4ah4lvp3ea02sbv"
+ keyword = "spoonacular"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSpoonacular_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Spoonacular,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword spoonacular",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a spoonacular secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Spoonacular,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Spoonacular.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Spoonacular.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sportsmonk/sportsmonk_integration_test.go b/pkg/detectors/sportsmonk/sportsmonk_integration_test.go
new file mode 100644
index 000000000..399df68b6
--- /dev/null
+++ b/pkg/detectors/sportsmonk/sportsmonk_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package sportsmonk
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSportsmonk_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SPORTSMONK")
+ inactiveSecret := testSecrets.MustGetField("SPORTSMONK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sportsmonk,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sportsmonk,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sportsmonk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sportsmonk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sportsmonk/sportsmonk_test.go b/pkg/detectors/sportsmonk/sportsmonk_test.go
index a04837313..4bf2390e4 100644
--- a/pkg/detectors/sportsmonk/sportsmonk_test.go
+++ b/pkg/detectors/sportsmonk/sportsmonk_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package sportsmonk
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSportsmonk_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SPORTSMONK")
- inactiveSecret := testSecrets.MustGetField("SPORTSMONK_INACTIVE")
+var (
+ validPattern = "GxSYzYakznVR2ZSu7cdIGjj2g3m4VD7vkhToot3WKwEJlgibw4a6akEeV55V"
+ invalidPattern = "GxSYzYakznVR2ZSu?cdIGjj2g3m4VD7vkhToot3WKwEJlgibw4a6akEeV55V"
+ keyword = "sportsmonk"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSportsmonk_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sportsmonk,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sportsmonk",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sportsmonk secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sportsmonk,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sportsmonk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Sportsmonk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/spotifykey/spotifykey_integration_test.go b/pkg/detectors/spotifykey/spotifykey_integration_test.go
new file mode 100644
index 000000000..a38741bfa
--- /dev/null
+++ b/pkg/detectors/spotifykey/spotifykey_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package spotifykey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSpotifyKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ clientID := testSecrets.MustGetField("SPOTIFY_CLIENTID")
+ clientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_SECRET")
+ inactiveClientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s", clientSecret, clientID)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SpotifyKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s but not valid", inactiveClientSecret, clientID)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SpotifyKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SpotifyKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SpotifyKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/spotifykey/spotifykey_test.go b/pkg/detectors/spotifykey/spotifykey_test.go
index 3aff9fe77..8999a08ba 100644
--- a/pkg/detectors/spotifykey/spotifykey_test.go
+++ b/pkg/detectors/spotifykey/spotifykey_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package spotifykey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSpotifyKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- clientID := testSecrets.MustGetField("SPOTIFY_CLIENTID")
- clientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_SECRET")
- inactiveClientSecret := testSecrets.MustGetField("SPOTIFY_CLIENT_INACTIVE")
+var (
+ validSecret = "MJsy6euxw0GKljGYZLZGnn0PvKuZB0V8"
+ invalidSecret = "MJsy6euxw0GKljGY?LZGnn0PvKuZB0V8"
+ validId = "JykXADk7r8ZegkS0emhtpFSWNqGVAIgm"
+ invalidId = "JykXADk7r8Z?gkS0emhtpFSWNqGVAIgm"
+ keyword = "spotifykey"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSpotifyKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s", clientSecret, clientID)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SpotifyKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword spotifykey",
+ input: fmt.Sprintf("%s key - '%s'\n%s id - '%s'\n", keyword, validSecret, keyword, validId),
+ want: []string{validSecret, validId},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a spotifyKey secret %s within spotifyId %s but not valid", inactiveClientSecret, clientID)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SpotifyKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s key - '%s'\n%s id - '%s'\n", keyword, invalidSecret, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SpotifyKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SpotifyKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sqlserver/sqlserver_test.go b/pkg/detectors/sqlserver/sqlserver_test.go
index 33b5fffe2..fc5748b71 100644
--- a/pkg/detectors/sqlserver/sqlserver_test.go
+++ b/pkg/detectors/sqlserver/sqlserver_test.go
@@ -4,7 +4,7 @@ import (
"testing"
)
-func TestSQLServer_pattern(t *testing.T) {
+func TestSQLServer_Pattern(t *testing.T) {
if !pattern.Match([]byte(`builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`)) {
t.Errorf("SQLServer.pattern: did not find connection string from Program.cs")
}
diff --git a/pkg/detectors/square/square_integration_test.go b/pkg/detectors/square/square_integration_test.go
new file mode 100644
index 000000000..d6b7793d4
--- /dev/null
+++ b/pkg/detectors/square/square_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package square
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSquare_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SQUARE_SECRET")
+ secretInactive := testSecrets.MustGetField("SQUARE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unvierifed",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a square secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Square.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ got[i].AnalysisInfo = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Square.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/square/square_test.go b/pkg/detectors/square/square_test.go
index 57d5fa21b..dadda930f 100644
--- a/pkg/detectors/square/square_test.go
+++ b/pkg/detectors/square/square_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package square
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSquare_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SQUARE_SECRET")
- secretInactive := testSecrets.MustGetField("SQUARE_INACTIVE")
+var (
+ validPattern = "EAAAYmTgYL4kQ-65xyZ4zANgipYVJQFTrb=roK23I=iFebzL4rjYbUg10N6o1l--"
+ invalidPattern = "EAAAYmTgYL4kQ-65xyZ4zANgipYVJQFT?b=roK23I=iFebzL4rjYbUg10N6o1l--"
+ keyword = "square"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSquare_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a square secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Square,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword square",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unvierifed",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a square secret %s within", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Square,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Square.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
- got[i].AnalysisInfo = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Square.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/squareapp/squareapp_integration_test.go b/pkg/detectors/squareapp/squareapp_integration_test.go
new file mode 100644
index 000000000..e36008c59
--- /dev/null
+++ b/pkg/detectors/squareapp/squareapp_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package squareapp
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSquareApp_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SQUAREAPP_SECRET")
+ secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")
+ id := testSecrets.MustGetField("SQUAREAPP_ID")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: true,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Square,
+ Verified: false,
+ Redacted: id,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SquareApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SquareApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/squareapp/squareapp_test.go b/pkg/detectors/squareapp/squareapp_test.go
index ad9874c6f..41918ed8c 100644
--- a/pkg/detectors/squareapp/squareapp_test.go
+++ b/pkg/detectors/squareapp/squareapp_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package squareapp
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSquareApp_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SQUAREAPP_SECRET")
- secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")
- id := testSecrets.MustGetField("SQUAREAPP_ID")
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validKey = "YJsq0ige-a9khwVJOSwlzBvX0wp4j8t90s2d"
+ invalidKey = "YJsq0ige-a9kh?VJOSwlzBvX0wp4j8t90s2d"
+ validSec = "4sSPeeM_jk0VZiTFZJqEwzvXHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"
+ invalidSec = "4sSPeeM_jk0VZiTFZJqEwz?XHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"
+ keyword = "squareapp"
+)
+
+func TestSquareApp_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Square,
- Verified: true,
- Redacted: id,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword squareapp",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSec),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Square,
- Verified: false,
- Redacted: id,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSec),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SquareApp.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SquareApp.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/squarespace/squarespace_integration_test.go b/pkg/detectors/squarespace/squarespace_integration_test.go
new file mode 100644
index 000000000..80308cb71
--- /dev/null
+++ b/pkg/detectors/squarespace/squarespace_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package squarespace
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSquarespace_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SQUARESPACE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SQUARESPACE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squarespace secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Squarespace,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squarespace secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Squarespace,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Squarespace.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Squarespace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/squarespace/squarespace_test.go b/pkg/detectors/squarespace/squarespace_test.go
index 1874e5816..ff66e179a 100644
--- a/pkg/detectors/squarespace/squarespace_test.go
+++ b/pkg/detectors/squarespace/squarespace_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package squarespace
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSquarespace_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SQUARESPACE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SQUARESPACE_INACTIVE")
+var (
+ validPattern = "9308f78b-8315-a8a6-9c9c-a4b41488508f"
+ invalidPattern = "9308f78b?8315-a8a6-9c9c-a4b41488508f"
+ keyword = "squarespace"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSquarespace_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a squarespace secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Squarespace,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword squarespace",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a squarespace secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Squarespace,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Squarespace.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Squarespace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/squareup/squareup_integration_test.go b/pkg/detectors/squareup/squareup_integration_test.go
new file mode 100644
index 000000000..fce1cce50
--- /dev/null
+++ b/pkg/detectors/squareup/squareup_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package squareup
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSquareup_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SQUAREUP")
+ inactiveSecret := testSecrets.MustGetField("SQUAREUP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squareup secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Squareup,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a squareup secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Squareup,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Squareup.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Squareup.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/squareup/squareup_test.go b/pkg/detectors/squareup/squareup_test.go
index fce1cce50..787eee9c9 100644
--- a/pkg/detectors/squareup/squareup_test.go
+++ b/pkg/detectors/squareup/squareup_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package squareup
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSquareup_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SQUAREUP")
- inactiveSecret := testSecrets.MustGetField("SQUAREUP_INACTIVE")
+var (
+ validPattern = "sq0idp-CIlelDA4a0mIksYXPGwUzy"
+ invalidPattern = "sq0idp-CIlelDA?a0mIksYXPGwUzy"
+ keyword = "squareup"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSquareup_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a squareup secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Squareup,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a squareup secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Squareup,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword squareup",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Squareup.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Squareup.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sslmate/sslmate_integration_test.go b/pkg/detectors/sslmate/sslmate_integration_test.go
new file mode 100644
index 000000000..7c5f39660
--- /dev/null
+++ b/pkg/detectors/sslmate/sslmate_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package sslmate
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSslMate_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SSLMATE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SSLMATE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sslmate secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SslMate,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sslmate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SslMate,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SslMate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SslMate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sslmate/sslmate_test.go b/pkg/detectors/sslmate/sslmate_test.go
index 8244136b1..ee9cf6e14 100644
--- a/pkg/detectors/sslmate/sslmate_test.go
+++ b/pkg/detectors/sslmate/sslmate_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package sslmate
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSslMate_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SSLMATE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SSLMATE_INACTIVE")
+var (
+ validPattern = "4RCKoI9YgcI43rpDbhyYpGdHtNaG1DLcwEYR"
+ invalidPattern = "4RCKoI9YgcI43rpDbh?YpGdHtNaG1DLcwEYR"
+ keyword = "sslmate"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSslMate_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sslmate secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SslMate,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sslmate",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sslmate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SslMate,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SslMate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SslMate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/statuscake/statuscake_integration_test.go b/pkg/detectors/statuscake/statuscake_integration_test.go
new file mode 100644
index 000000000..8cd8609cf
--- /dev/null
+++ b/pkg/detectors/statuscake/statuscake_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package statuscake
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStatuscake_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STATUSCAKE")
+ inactiveSecret := testSecrets.MustGetField("STATUSCAKE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a statuscake secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Statuscake,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a statuscake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Statuscake,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Statuscake.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Statuscake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/statuscake/statuscake_test.go b/pkg/detectors/statuscake/statuscake_test.go
index 291347382..041ae4efe 100644
--- a/pkg/detectors/statuscake/statuscake_test.go
+++ b/pkg/detectors/statuscake/statuscake_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package statuscake
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStatuscake_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STATUSCAKE")
- inactiveSecret := testSecrets.MustGetField("STATUSCAKE_INACTIVE")
+var (
+ validPattern = "jwI4HrRAbQV4rUYYD76C"
+ invalidPattern = "jwI4HrRAbQ?4rUYYD76C"
+ keyword = "statuscake"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStatuscake_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a statuscake secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Statuscake,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword statuscake",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a statuscake secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Statuscake,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Statuscake.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Statuscake.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/statuspage/statuspage_integration_test.go b/pkg/detectors/statuspage/statuspage_integration_test.go
new file mode 100644
index 000000000..c2c8335dc
--- /dev/null
+++ b/pkg/detectors/statuspage/statuspage_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package statuspage
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStatuspage_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STATUSPAGE")
+ inactiveSecret := testSecrets.MustGetField("STATUSPAGE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a statuspage secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Statuspage,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a statuspage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Statuspage,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Statuspage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Statuspage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/statuspage/statuspage_test.go b/pkg/detectors/statuspage/statuspage_test.go
index f66e0a71d..197ea035d 100644
--- a/pkg/detectors/statuspage/statuspage_test.go
+++ b/pkg/detectors/statuspage/statuspage_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package statuspage
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStatuspage_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STATUSPAGE")
- inactiveSecret := testSecrets.MustGetField("STATUSPAGE_INACTIVE")
+var (
+ validPattern = "0c8xiakjyy42sep7fs17pzq08ra0e4aj04uq"
+ invalidPattern = "0c8xiakjyy42sep7fs?7pzq08ra0e4aj04uq"
+ keyword = "statuspage"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStatuspage_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a statuspage secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Statuspage,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword statuspage",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a statuspage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Statuspage,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Statuspage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Statuspage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/statuspal/statuspal_integration_test.go b/pkg/detectors/statuspal/statuspal_integration_test.go
new file mode 100644
index 000000000..2f1a9547b
--- /dev/null
+++ b/pkg/detectors/statuspal/statuspal_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package statuspal
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStatuspal_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STATUSPAL")
+ inactiveSecret := testSecrets.MustGetField("STATUSPAL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a statuspal secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Statuspal,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a statuspal secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Statuspal,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Statuspal.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Statuspal.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/statuspal/statuspal_test.go b/pkg/detectors/statuspal/statuspal_test.go
index 8f43e6141..10fcfee9a 100644
--- a/pkg/detectors/statuspal/statuspal_test.go
+++ b/pkg/detectors/statuspal/statuspal_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package statuspal
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStatuspal_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STATUSPAL")
- inactiveSecret := testSecrets.MustGetField("STATUSPAL_INACTIVE")
+var (
+ validPattern = "FIYuLMuzF8XPqwPQzEiQFzJziMhcrAfH"
+ invalidPattern = "FIYuLMu?F8XPqwPQzEiQFzJziMhcrAfH"
+ keyword = "statuspal"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStatuspal_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a statuspal secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Statuspal,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword statuspal",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a statuspal secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Statuspal,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Statuspal.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Statuspal.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/stitchdata/stitchdata_integration_test.go b/pkg/detectors/stitchdata/stitchdata_integration_test.go
new file mode 100644
index 000000000..e3f0c9a3f
--- /dev/null
+++ b/pkg/detectors/stitchdata/stitchdata_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package stitchdata
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStitchdata_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STITCHDATA")
+ inactiveSecret := testSecrets.MustGetField("STITCHDATA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stitchdata,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stitchdata,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stitchdata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stitchdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stitchdata/stitchdata_test.go b/pkg/detectors/stitchdata/stitchdata_test.go
index 42120ff03..fc3d40cd0 100644
--- a/pkg/detectors/stitchdata/stitchdata_test.go
+++ b/pkg/detectors/stitchdata/stitchdata_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package stitchdata
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStitchdata_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STITCHDATA")
- inactiveSecret := testSecrets.MustGetField("STITCHDATA_INACTIVE")
+var (
+ validPattern = "8n4bfv0w_ygprt3xkr3gz1ez6jmz0oo9eva"
+ invalidPattern = "8n4bfv0w_ygp?t3xkr3gz1ez6jmz0oo9eva"
+ keyword = "stitchdata"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStitchdata_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stitchdata,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword stitchdata",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stitchdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stitchdata,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stitchdata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Stitchdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/stockdata/stockdata_integration_test.go b/pkg/detectors/stockdata/stockdata_integration_test.go
new file mode 100644
index 000000000..d7794858e
--- /dev/null
+++ b/pkg/detectors/stockdata/stockdata_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package stockdata
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStockdata_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STOCKDATA")
+ inactiveSecret := testSecrets.MustGetField("STOCKDATA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stockdata secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stockdata,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stockdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stockdata,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stockdata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stockdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stockdata/stockdata_test.go b/pkg/detectors/stockdata/stockdata_test.go
index dd4654864..04bb01fef 100644
--- a/pkg/detectors/stockdata/stockdata_test.go
+++ b/pkg/detectors/stockdata/stockdata_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package stockdata
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStockdata_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STOCKDATA")
- inactiveSecret := testSecrets.MustGetField("STOCKDATA_INACTIVE")
+var (
+ validPattern = "94ny9HMpy5rGR7jI3Z2fjBQpuU8gQofhlw4u3sRr"
+ invalidPattern = "94ny9HMpy5rGR7?I3Z2fjBQpuU8gQofhlw4u3sRr"
+ keyword = "stockdata"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStockdata_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stockdata secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stockdata,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword stockdata",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stockdata secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stockdata,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stockdata.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Stockdata.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/storecove/storecove_integration_test.go b/pkg/detectors/storecove/storecove_integration_test.go
new file mode 100644
index 000000000..5adfc8b7e
--- /dev/null
+++ b/pkg/detectors/storecove/storecove_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package storecove
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStorecove_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STORECOVE")
+ inactiveSecret := testSecrets.MustGetField("STORECOVE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a storecove secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Storecove,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a storecove secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Storecove,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Storecove.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Storecove.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/storecove/storecove_test.go b/pkg/detectors/storecove/storecove_test.go
index 42013daad..7fbabecd9 100644
--- a/pkg/detectors/storecove/storecove_test.go
+++ b/pkg/detectors/storecove/storecove_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package storecove
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStorecove_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STORECOVE")
- inactiveSecret := testSecrets.MustGetField("STORECOVE_INACTIVE")
+var (
+ validPattern = "yDDqj4B00b1m_HymYpqVRbPHWi7njN60NYsT0tpHvEe"
+ invalidPattern = "yDDqj4B00?1m_HymYpqVRbPHWi7njN60NYsT0tpHvEe"
+ keyword = "storecove"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStorecove_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a storecove secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Storecove,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword storecove",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a storecove secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Storecove,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Storecove.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Storecove.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/stormboard/stormboard_integration_test.go b/pkg/detectors/stormboard/stormboard_integration_test.go
new file mode 100644
index 000000000..cf537ef23
--- /dev/null
+++ b/pkg/detectors/stormboard/stormboard_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package stormboard
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStormboard_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STORMBOARD")
+ inactiveSecret := testSecrets.MustGetField("STORMBOARD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stormboard secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stormboard,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stormboard secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stormboard,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stormboard.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stormboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stormboard/stormboard_test.go b/pkg/detectors/stormboard/stormboard_test.go
index 2f1618fa1..ecc3d0fc6 100644
--- a/pkg/detectors/stormboard/stormboard_test.go
+++ b/pkg/detectors/stormboard/stormboard_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package stormboard
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStormboard_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STORMBOARD")
- inactiveSecret := testSecrets.MustGetField("STORMBOARD_INACTIVE")
+var (
+ validPattern = "46e7fa9b370dd56209a5610630a45bb9a54fa1e9"
+ invalidPattern = "4?e7fa9b370dd56209a5610630a45bb9a54fa1e9"
+ keyword = "stormboard"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStormboard_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stormboard secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stormboard,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword stormboard",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stormboard secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stormboard,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stormboard.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Stormboard.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/stormglass/stormglass_integration_test.go b/pkg/detectors/stormglass/stormglass_integration_test.go
new file mode 100644
index 000000000..8bdb9dfb8
--- /dev/null
+++ b/pkg/detectors/stormglass/stormglass_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package stormglass
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStormglass_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STORMGLASS")
+ inactiveSecret := testSecrets.MustGetField("STORMGLASS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stormglass secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stormglass,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stormglass secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stormglass,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stormglass.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stormglass.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stormglass/stormglass_test.go b/pkg/detectors/stormglass/stormglass_test.go
index d4474a05f..26198eee3 100644
--- a/pkg/detectors/stormglass/stormglass_test.go
+++ b/pkg/detectors/stormglass/stormglass_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package stormglass
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStormglass_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STORMGLASS")
- inactiveSecret := testSecrets.MustGetField("STORMGLASS_INACTIVE")
+var (
+ validPattern = "1sijzovx0w8ru40qteilgbxej6a5A1moc7s9b1p3mx9472dho5svv5yA7b0aqzu-Ai-z-r1xz"
+ invalidPattern = "1sijzovx0w8ru40qteilg?xej6a5A1moc7s9b1p3mx9472dho5svv5yA7b0aqzu-Ai-z-r1xz"
+ keyword = "stormglass"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStormglass_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stormglass secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stormglass,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword stormglass",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stormglass secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stormglass,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stormglass.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Stormglass.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/storyblok/storyblok_integration_test.go b/pkg/detectors/storyblok/storyblok_integration_test.go
new file mode 100644
index 000000000..0320129eb
--- /dev/null
+++ b/pkg/detectors/storyblok/storyblok_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package storyblok
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStoryblok_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STORYBLOK_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("STORYBLOK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a storyblok secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Storyblok,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a storyblok secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Storyblok,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Storyblok.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Storyblok.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/storyblok/storyblok_test.go b/pkg/detectors/storyblok/storyblok_test.go
index 85c41035a..3afcca6b3 100644
--- a/pkg/detectors/storyblok/storyblok_test.go
+++ b/pkg/detectors/storyblok/storyblok_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package storyblok
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStoryblok_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STORYBLOK_TOKEN")
- inactiveSecret := testSecrets.MustGetField("STORYBLOK_INACTIVE")
+var (
+ validPattern = "ZXIic758xXghN5SOkRI57ytt"
+ invalidPattern = "ZXIic758xXgh?5SOkRI57ytt"
+ keyword = "storyblok"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStoryblok_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a storyblok secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Storyblok,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword storyblok",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a storyblok secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Storyblok,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Storyblok.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Storyblok.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/storychief/storychief_integration_test.go b/pkg/detectors/storychief/storychief_integration_test.go
new file mode 100644
index 000000000..dade17d2a
--- /dev/null
+++ b/pkg/detectors/storychief/storychief_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package storychief
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStorychief_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STORYCHIEF")
+ inactiveSecret := testSecrets.MustGetField("STORYCHIEF_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a storychief secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Storychief,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a storychief secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Storychief,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Storychief.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Storychief.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/storychief/storychief_test.go b/pkg/detectors/storychief/storychief_test.go
index dade17d2a..a03e5c203 100644
--- a/pkg/detectors/storychief/storychief_test.go
+++ b/pkg/detectors/storychief/storychief_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package storychief
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStorychief_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STORYCHIEF")
- inactiveSecret := testSecrets.MustGetField("STORYCHIEF_INACTIVE")
+var (
+ validPattern = "OjhRmEboKs5kYVWJOrpTrloSRsAdkEl8PK-eHrSID5TRR.59-m5ezCY1dcJqJCxbV.2.LeGuqnhuouYkaiJ-iq90X-XX0J3IcT-ReLn.lAU0FfkvHOoOFp4k8w0-nDKAI8irzT5.pi7bmathhUdZO40-Rb59B6M0h40LbAkcvW49YP_-xXqGV_s.tCRbbzeUWt.Y9cFzfrQfRaVlTTqF5AC4mTEF4UxCSHK7uEE3OdzKhRehslviyKozUqer9HEZQ941rCqeEpt8kDcC0GOZFskrqu-EynCRJRhK6cv682e3HoRqE9x5FtcJ3gPfJEA70yHF5gTT0gh1KYDPKoJ-SQ7cjEBxxn-NbLu_WD3HW8DsjVDu1MgqrjqrnFNQvEAqYM7RI.RurVC38TP-5PioJqzJbvpNzkvwFjDrPpHdcivLgDEJUXS39PkIZPb1WCk.LPMOj5MBAwgn7TuADlV34Tael2FxygTPzA6ZVHBzm4aoUZ94Fwcm1vAkKJPeydsu33lmJ73e55pp7IhFyRO6MdPgLHqm3XkUhleU4yDUAEPWMyhilzOEO1t3nP3plfPjZU1.A1VEgWoOjvhs61qAMj2O6YsVFc7nU-PRlOpqj7yJNRmLWnJGVZW1UQLYwo5urJTb92u8BBPe179Eldzk5-xQ994NnrROnAK5DXkwdw9KII85fVBof8LGei1ocVEzYodTVvVY75iXaVmIP3Sf3dqkumW9Jdik-G-Lz.tvyJMTUSmtbX1oXaByyInng89h0Ah1O7D36nUm-gSOOgjJAssWCF0jiOSb2ps7BCdArjd5BVEOhewpodrtDs.iOncs8dtMSmyA5N7Jhzo2eINenb9dhJ7yQmskzhQcN-jpHKLpiL.w4lqCZ5X.uI_oDjx6V_7bJQK07uWCEB8xiwTRCCnB0mZYmi5q0WpG4sCY2xIW"
+ invalidPattern = "O?hRmEboKs5kYVWJOrpTrloSRsAdkEl8PK-eHrSID5TRR.59-m5ezCY1dcJqJCxbV.2.LeGuqnhuouYkaiJ-iq90X-XX0J3IcT-ReLn.lAU0FfkvHOoOFp4k8w0-nDKAI8irzT5.pi7bmathhUdZO40-Rb59B6M0h40LbAkcvW49YP_-xXqGV_s.tCRbbzeUWt.Y9cFzfrQfRaVlTTqF5AC4mTEF4UxCSHK7uEE3OdzKhRehslviyKozUqer9HEZQ941rCqeEpt8kDcC0GOZFskrqu-EynCRJRhK6cv682e3HoRqE9x5FtcJ3gPfJEA70yHF5gTT0gh1KYDPKoJ-SQ7cjEBxxn-NbLu_WD3HW8DsjVDu1MgqrjqrnFNQvEAqYM7RI.RurVC38TP-5PioJqzJbvpNzkvwFjDrPpHdcivLgDEJUXS39PkIZPb1WCk.LPMOj5MBAwgn7TuADlV34Tael2FxygTPzA6ZVHBzm4aoUZ94Fwcm1vAkKJPeydsu33lmJ73e55pp7IhFyRO6MdPgLHqm3XkUhleU4yDUAEPWMyhilzOEO1t3nP3plfPjZU1.A1VEgWoOjvhs61qAMj2O6YsVFc7nU-PRlOpqj7yJNRmLWnJGVZW1UQLYwo5urJTb92u8BBPe179Eldzk5-xQ994NnrROnAK5DXkwdw9KII85fVBof8LGei1ocVEzYodTVvVY75iXaVmIP3Sf3dqkumW9Jdik-G-Lz.tvyJMTUSmtbX1oXaByyInng89h0Ah1O7D36nUm-gSOOgjJAssWCF0jiOSb2ps7BCdArjd5BVEOhewpodrtDs.iOncs8dtMSmyA5N7Jhzo2eINenb9dhJ7yQmskzhQcN-jpHKLpiL.w4lqCZ5X.uI_oDjx6V_7bJQK07uWCEB8xiwTRCCnB0mZYmi5q0WpG4sCY2xIW"
+ keyword = "storychief"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStorychief_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a storychief secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Storychief,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword storychief",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
+ },
+ {
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a storychief secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Storychief,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Storychief.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Storychief.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/strava/strava_integration_test.go b/pkg/detectors/strava/strava_integration_test.go
new file mode 100644
index 000000000..908965540
--- /dev/null
+++ b/pkg/detectors/strava/strava_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package strava
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStrava_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("STRAVA_ID")
+ secret := testSecrets.MustGetField("STRAVA_SECRET")
+ token := testSecrets.MustGetField("STRAVA_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("STRAVA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s", secret, id, token)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Strava,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s but not valid", inactiveSecret, id, token)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Strava,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Strava.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Strava.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/strava/strava_test.go b/pkg/detectors/strava/strava_test.go
index 93b10ddea..204f91382 100644
--- a/pkg/detectors/strava/strava_test.go
+++ b/pkg/detectors/strava/strava_test.go
@@ -1,122 +1,85 @@
-//go:build detectors
-// +build detectors
-
package strava
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStrava_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("STRAVA_ID")
- secret := testSecrets.MustGetField("STRAVA_SECRET")
- token := testSecrets.MustGetField("STRAVA_TOKEN")
- inactiveSecret := testSecrets.MustGetField("STRAVA_INACTIVE")
+var (
+ validId = "57732"
+ invalidId = "5?732"
+ validSecret = "ateo889bcja9fyt4gff2kqe187uhy8l0il4vf2cw"
+ invalidSecret = "ateo889bcja9fyt4gff2?qe187uhy8l0il4vf2cw"
+ validKey = "e8qmzsdc0qbq4hsal5m4q1pz5s2yloydbl1bjc1u"
+ invalidKey = "e8?mzsdc0qbq4hsal5m4q1pz5s2yloydbl1bjc1u"
+ keyword = "strava"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStrava_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s", secret, id, token)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Strava,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword strava",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, validId, keyword, validSecret, keyword, validKey),
+ want: []string{validId + validSecret, validId + validKey, validId + validKey, validId + validSecret},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a strava secret %s within strava id %s and strava token %s but not valid", inactiveSecret, id, token)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Strava,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n%s token - '%s'\n", keyword, invalidId, keyword, invalidSecret, keyword, invalidKey),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Strava.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Strava.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/streak/streak_integration_test.go b/pkg/detectors/streak/streak_integration_test.go
new file mode 100644
index 000000000..38c4e7d45
--- /dev/null
+++ b/pkg/detectors/streak/streak_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package streak
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStreak_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STREAK")
+ inactiveSecret := testSecrets.MustGetField("STREAK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a streak secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Streak,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a streak secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Streak,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Streak.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Streak.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/streak/streak_test.go b/pkg/detectors/streak/streak_test.go
index a9d256ae0..ea1ffeff4 100644
--- a/pkg/detectors/streak/streak_test.go
+++ b/pkg/detectors/streak/streak_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package streak
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStreak_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STREAK")
- inactiveSecret := testSecrets.MustGetField("STREAK_INACTIVE")
+var (
+ validPattern = "bf1949fA2574fb6b2deb94AdA51f0719"
+ invalidPattern = "bf1949fA?574fb6b2deb94AdA51f0719"
+ keyword = "streak"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStreak_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a streak secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Streak,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword streak",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a streak secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Streak,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Streak.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Streak.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/stripe/stripe_integration_test.go b/pkg/detectors/stripe/stripe_integration_test.go
new file mode 100644
index 000000000..19bb0836a
--- /dev/null
+++ b/pkg/detectors/stripe/stripe_integration_test.go
@@ -0,0 +1,126 @@
+//go:build detectors
+// +build detectors
+
+package stripe
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStripe_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STRIPE_SECRET")
+ secretInactive := testSecrets.MustGetField("STRIPE_INACTIVE")
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stripe,
+ Verified: true,
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stripe,
+ Verified: false,
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stripe.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ got[i].AnalysisInfo = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stripe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stripe/stripe_test.go b/pkg/detectors/stripe/stripe_test.go
index 905a0d721..5c6c9a046 100644
--- a/pkg/detectors/stripe/stripe_test.go
+++ b/pkg/detectors/stripe/stripe_test.go
@@ -1,125 +1,81 @@
-//go:build detectors
-// +build detectors
-
package stripe
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStripe_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STRIPE_SECRET")
- secretInactive := testSecrets.MustGetField("STRIPE_INACTIVE")
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+var (
+ validPattern = "rk_live_HUOlkIKhNEYOPS0oDSwGwJHbg4xGaXNeJZ2CdvDGeVZQHljoq5TuFwQHgME3W"
+ invalidPattern = "?k_live_HUOlkIKhNEYOPS0oDSwGwJHbg4xGaXNeJZ2CdvDGeVZQHljoq5TuFwQHgME3?"
+ keyword = "stripe"
+)
+
+func TestStripe_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stripe,
- Verified: true,
- ExtraData: map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",
- },
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword stripe",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stripe secret %s within", secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stripe,
- Verified: false,
- ExtraData: map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/stripe/",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stripe.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
- got[i].AnalysisInfo = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Stripe.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/stripo/stripo_integration_test.go b/pkg/detectors/stripo/stripo_integration_test.go
new file mode 100644
index 000000000..ae2bb3c72
--- /dev/null
+++ b/pkg/detectors/stripo/stripo_integration_test.go
@@ -0,0 +1,127 @@
+//go:build detectors
+// +build detectors
+
+package stripo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStripo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STRIPO")
+ inactiveSecret := testSecrets.MustGetField("STRIPO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stripo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stripo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stripo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stripo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stripo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Stripo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stripo/stripo_test.go b/pkg/detectors/stripo/stripo_test.go
index 3c6671873..fd66f01bd 100644
--- a/pkg/detectors/stripo/stripo_test.go
+++ b/pkg/detectors/stripo/stripo_test.go
@@ -1,127 +1,90 @@
-//go:build detectors
-// +build detectors
-
package stripo
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStripo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STRIPO")
- inactiveSecret := testSecrets.MustGetField("STRIPO_INACTIVE")
+var (
+ validPattern = "eyJhbGciOiJIUzI1NiJ9.qflXV62P6Y51gAFD9pnq9FpcgDQOfr68GtBUARUvk7gzbAfg6eF66Txb4R9idT41JsG5CqTKXMfMnzpPVByJRFjtfXIFasywYCd53PqB774bhbBFgfEznjdQjOu8ui910q.8QQbrVsL16ya98sn8yyjLbBX21bix2G_QhFg_rKN9Bv"
+ invalidPattern = "ayJhbGciOiJIUzI1NiJ9.qflXV62P6Y51gAFD9pnq9FpcgDQOfr68GtBUARUvk7gzbAfg6eF66Txb4R9idT41JsG5CqTKXMfMnzpPVByJRFjtfXIFasywYCd53PqB774bhbBFgfEznjdQjOu8ui910q.8QQbrVsL16ya98sn8yyjLbBX21bix2G_QhFg_rKN9Bv"
+ keyword = "stripo"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStripo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stripo secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stripo,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - with keyword stripo",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stripo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stripo,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stripo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Stripo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/stytch/stytch_integration_test.go b/pkg/detectors/stytch/stytch_integration_test.go
new file mode 100644
index 000000000..6de25ac8a
--- /dev/null
+++ b/pkg/detectors/stytch/stytch_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package stytch
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestStytch_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("STYTCH")
+ id := testSecrets.MustGetField("STYTCH_ID")
+ inactiveSecret := testSecrets.MustGetField("STYTCH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stytch,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Stytch,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Stytch.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Stytch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/stytch/stytch_test.go b/pkg/detectors/stytch/stytch_test.go
index 0d1722dae..5a6d09c3e 100644
--- a/pkg/detectors/stytch/stytch_test.go
+++ b/pkg/detectors/stytch/stytch_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package stytch
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestStytch_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("STYTCH")
- id := testSecrets.MustGetField("STYTCH_ID")
- inactiveSecret := testSecrets.MustGetField("STYTCH_INACTIVE")
+var (
+ validKey = "TSIDpkOi1MKTudKSFs-KXGdOeH3ECTeXWtX7OAr4Y9k31m8="
+ invalidKey = "TSIDpkOi1MKTudKSFs-KXGdO?H3ECTeXWtX7OAr4Y9k31m8="
+ validId = "k7hi7kqgb6a8rqi0b8nbna1p-o32rv9xzlnvzbd5oa9yr-vtp"
+ invalidId = "k7hi7kqgb6a8rqi0b8nbna1p?o32rv9xzlnvzbd5oa9yr-vtp"
+ keyword = "stytch"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestStytch_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stytch,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword stytch",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a stytch secret %s within stytch %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Stytch,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Stytch.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Stytch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/sugester/sugester_integration_test.go b/pkg/detectors/sugester/sugester_integration_test.go
new file mode 100644
index 000000000..4b27ef9a3
--- /dev/null
+++ b/pkg/detectors/sugester/sugester_integration_test.go
@@ -0,0 +1,118 @@
+//go:build detectors
+// +build detectors
+
+package sugester
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSugester_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SUGESTER")
+ domain := testSecrets.MustGetField("SUGESTER_DOMAIN")
+ inactiveSecret := testSecrets.MustGetField("SUGESTER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sugester,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Sugester,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Sugester.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Sugester.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/sugester/sugester_test.go b/pkg/detectors/sugester/sugester_test.go
index 4b27ef9a3..aa9d07dfa 100644
--- a/pkg/detectors/sugester/sugester_test.go
+++ b/pkg/detectors/sugester/sugester_test.go
@@ -1,117 +1,82 @@
-//go:build detectors
-// +build detectors
-
package sugester
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSugester_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SUGESTER")
- domain := testSecrets.MustGetField("SUGESTER_DOMAIN")
- inactiveSecret := testSecrets.MustGetField("SUGESTER_INACTIVE")
+var (
+ validKey = "RfQEMjlVIolNGUebkCS7iw0MevjgAD06"
+ invalidKey = "RfQEMjlVIolNGUeb?CS7iw0MevjgAD06"
+ validDomain = "h$6o0+jTX"
+ invalidDomain = "?$6o0+jT?"
+ keyword = "sugester"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSugester_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s", secret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sugester,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword sugester",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validDomain),
+ want: []string{validKey, validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a sugester secret %s within sugesterdomain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Sugester,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidDomain),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Sugester.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Sugester.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/sumologickey/sumologickey_integration_test.go b/pkg/detectors/sumologickey/sumologickey_integration_test.go
index e95e34041..4776f5b49 100644
--- a/pkg/detectors/sumologickey/sumologickey_integration_test.go
+++ b/pkg/detectors/sumologickey/sumologickey_integration_test.go
@@ -10,9 +10,9 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
diff --git a/pkg/detectors/supabasetoken/supabasetoken_integration_test.go b/pkg/detectors/supabasetoken/supabasetoken_integration_test.go
new file mode 100644
index 000000000..5c97a18ad
--- /dev/null
+++ b/pkg/detectors/supabasetoken/supabasetoken_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package supabasetoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSupabasetoken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SUPABASETOKEN")
+ inactiveSecret := testSecrets.MustGetField("SUPABASETOKEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SupabaseToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SupabaseToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Supabasetoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Supabasetoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/supabasetoken/supabasetoken_test.go b/pkg/detectors/supabasetoken/supabasetoken_test.go
index 840e235b8..dcdb9a406 100644
--- a/pkg/detectors/supabasetoken/supabasetoken_test.go
+++ b/pkg/detectors/supabasetoken/supabasetoken_test.go
@@ -1,121 +1,81 @@
-//go:build detectors
-// +build detectors
-
package supabasetoken
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSupabasetoken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SUPABASETOKEN")
- inactiveSecret := testSecrets.MustGetField("SUPABASETOKEN_INACTIVE")
+var (
+ validPattern = "sbp_6corwd8ltv0m2hx5av0bqswrldi43hk8qdtnxcs7"
+ invalidPattern = "sbp_6corwd8ltv?m2hx5av0bqswrldi43hk8qdtnxcs7"
+ keyword = "supabasetoken"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSupabasetoken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SupabaseToken,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword supabasetoken",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a supabasetoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SupabaseToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Supabasetoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Supabasetoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/supernotesapi/supernotesapi_integration_test.go b/pkg/detectors/supernotesapi/supernotesapi_integration_test.go
new file mode 100644
index 000000000..cf6a075a1
--- /dev/null
+++ b/pkg/detectors/supernotesapi/supernotesapi_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package supernotesapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSuperNotesAPI_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SUPERNOTESAPI")
+ inactiveSecret := testSecrets.MustGetField("SUPERNOTESAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a supernotes secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SuperNotesAPI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a supernotes secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SuperNotesAPI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SuperNotesAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SuperNotesAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/supernotesapi/supernotesapi_test.go b/pkg/detectors/supernotesapi/supernotesapi_test.go
index cf6a075a1..43e926d9a 100644
--- a/pkg/detectors/supernotesapi/supernotesapi_test.go
+++ b/pkg/detectors/supernotesapi/supernotesapi_test.go
@@ -1,116 +1,90 @@
-//go:build detectors
-// +build detectors
-
package supernotesapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSuperNotesAPI_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SUPERNOTESAPI")
- inactiveSecret := testSecrets.MustGetField("SUPERNOTESAPI_INACTIVE")
+var (
+ validPattern = "V77f640DIDmrPcYFMwCKPhAR7CkXDwXjmMGpzHVm6yC"
+ invalidPattern = "V77f640DIDmrPcYFMwCKPh?R7CkXDwXjmMGpzHVm6yC"
+ keyword = "supernotesapi"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSuperNotesAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a supernotes secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SuperNotesAPI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword supernotesapi",
+ input: fmt.Sprintf("%s token = ' %s '", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a supernotes secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SuperNotesAPI,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = ' %s ' | ' %s '", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = ' %s '", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = ' %s '", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SuperNotesAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SuperNotesAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/surveyanyplace/surveyanyplace_integration_test.go b/pkg/detectors/surveyanyplace/surveyanyplace_integration_test.go
new file mode 100644
index 000000000..6a849f3e8
--- /dev/null
+++ b/pkg/detectors/surveyanyplace/surveyanyplace_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package surveyanyplace
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSurveyAnyplace_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SURVEYANYPLACE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SURVEYANYPLACE_INACTIVE")
+ id := testSecrets.MustGetField("SURVEYANYPLACE_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SurveyAnyplace,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SurveyAnyplace,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SurveyAnyplace.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SurveyAnyplace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/surveyanyplace/surveyanyplace_test.go b/pkg/detectors/surveyanyplace/surveyanyplace_test.go
index 4f57a6215..538ec420d 100644
--- a/pkg/detectors/surveyanyplace/surveyanyplace_test.go
+++ b/pkg/detectors/surveyanyplace/surveyanyplace_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package surveyanyplace
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSurveyAnyplace_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SURVEYANYPLACE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SURVEYANYPLACE_INACTIVE")
- id := testSecrets.MustGetField("SURVEYANYPLACE_ID")
+var (
+ validKey = "zDbWB2TozuNJj8o56FIB0PVfE7LfjHFd"
+ invalidKey = "zDbWB2TozuNJj8o5?FIB0PVfE7LfjHFd"
+ validId = "UEN9VEUk4z1oY-pV2CPLZBAnn0rPWY3LCqNv"
+ invalidId = "UEN9VEUk4z1oY-pV2C?LZBAnn0rPWY3LCqNv"
+ keyword = "surveyanyplace"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSurveyAnyplace_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SurveyAnyplace,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword surveyanyplace",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a surveyanyplace secret %s within surveyanyplaceid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SurveyAnyplace,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SurveyAnyplace.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SurveyAnyplace.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/surveybot/surveybot_integration_test.go b/pkg/detectors/surveybot/surveybot_integration_test.go
new file mode 100644
index 000000000..dd567c2c1
--- /dev/null
+++ b/pkg/detectors/surveybot/surveybot_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package surveybot
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSurveyBot_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SURVEYBOT_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SURVEYBOT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a surveybot secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SurveyBot,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a surveybot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SurveyBot,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SurveyBot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SurveyBot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/surveybot/surveybot_test.go b/pkg/detectors/surveybot/surveybot_test.go
index 9fe32246e..c8210aca3 100644
--- a/pkg/detectors/surveybot/surveybot_test.go
+++ b/pkg/detectors/surveybot/surveybot_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package surveybot
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSurveyBot_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SURVEYBOT_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SURVEYBOT_INACTIVE")
+var (
+ validPattern = "ZGPxFPcng3xOumtiY14y8Gtp8IAp5pHlnxOf28l3ThncGCOHdU4o01muxq4UqAJP28s4hfkASbzVY0Qi"
+ invalidPattern = "ZGPxFPcng3xOumtiY14y8Gtp8IAp5pHlnxOf28l3?hncGCOHdU4o01muxq4UqAJP28s4hfkASbzVY0Qi"
+ keyword = "surveybot"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSurveyBot_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a surveybot secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SurveyBot,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword surveybot",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a surveybot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SurveyBot,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SurveyBot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SurveyBot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/surveysparrow/surveysparrow_integration_test.go b/pkg/detectors/surveysparrow/surveysparrow_integration_test.go
new file mode 100644
index 000000000..607b5a53f
--- /dev/null
+++ b/pkg/detectors/surveysparrow/surveysparrow_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package surveysparrow
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSurveySparrow_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SURVEYSPARROW_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SURVEYSPARROW_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SurveySparrow,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_SurveySparrow,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("SurveySparrow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("SurveySparrow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/surveysparrow/surveysparrow_test.go b/pkg/detectors/surveysparrow/surveysparrow_test.go
index 2a7c7be70..c82d88b03 100644
--- a/pkg/detectors/surveysparrow/surveysparrow_test.go
+++ b/pkg/detectors/surveysparrow/surveysparrow_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package surveysparrow
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSurveySparrow_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SURVEYSPARROW_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SURVEYSPARROW_INACTIVE")
+var (
+ validPattern = "nRTqAmYY7E8pi6CaSf4JGgvxreYC-DEI6Q9De1EYLilWDuSHGOugKnWYdIz7ifGlFDol4v2hsYH7C9lu5Z63XmwP"
+ invalidPattern = "nRTqAmYY7E8pi6CaSf4JGgvxreYC-?EI6Q9De1EYLilWDuSHGOugKnWYdIz7ifGlFDol4v2hsYH7C9lu5Z63XmwP"
+ keyword = "surveysparrow"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSurveySparrow_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SurveySparrow,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword surveysparrow",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a surveysparrow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_SurveySparrow,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("SurveySparrow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("SurveySparrow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/survicate/survicate_integration_test.go b/pkg/detectors/survicate/survicate_integration_test.go
new file mode 100644
index 000000000..61db64034
--- /dev/null
+++ b/pkg/detectors/survicate/survicate_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package survicate
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSurvicate_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SURVICATE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("SURVICATE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a survicate secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Survicate,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a survicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Survicate,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Survicate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Survicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/survicate/survicate_test.go b/pkg/detectors/survicate/survicate_test.go
index c9c93ca73..f4e2c9370 100644
--- a/pkg/detectors/survicate/survicate_test.go
+++ b/pkg/detectors/survicate/survicate_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package survicate
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSurvicate_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SURVICATE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("SURVICATE_INACTIVE")
+var (
+ validPattern = "lztej917o59y8x82f311v8d684eb2t6k"
+ invalidPattern = "lztej917o59y8x82?311v8d684eb2t6k"
+ keyword = "survicate"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSurvicate_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a survicate secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Survicate,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword survicate",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a survicate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Survicate,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Survicate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Survicate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/swell/swell_integration_test.go b/pkg/detectors/swell/swell_integration_test.go
new file mode 100644
index 000000000..90edf0702
--- /dev/null
+++ b/pkg/detectors/swell/swell_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package swell
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSwell_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SWELL")
+ id := testSecrets.MustGetField("SWELL_ID")
+ inactiveSecret := testSecrets.MustGetField("SWELL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Swell,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Swell,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Swell.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Swell.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/swell/swell_test.go b/pkg/detectors/swell/swell_test.go
index 1994ea923..00a6ef92c 100644
--- a/pkg/detectors/swell/swell_test.go
+++ b/pkg/detectors/swell/swell_test.go
@@ -1,121 +1,83 @@
-//go:build detectors
-// +build detectors
-
package swell
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSwell_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SWELL")
- id := testSecrets.MustGetField("SWELL_ID")
- inactiveSecret := testSecrets.MustGetField("SWELL_INACTIVE")
+var (
+ validKey = "Mkff1bQx72mxhvNdCsudapMP7eery91G"
+ invalidKey = "Mkff1bQx72mxhvNd?sudapMP7eery91G"
+ validId = "0aOWuIMHYK6F30I1"
+ invalidId = "?aOWuIMHYK6F30I?"
+ keyword = "swell"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSwell_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Swell,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword swell",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validId),
+ want: []string{validKey},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a swell secret %s within swell %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Swell,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidId),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Swell.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Swell.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/swiftype/swiftype_integration_test.go b/pkg/detectors/swiftype/swiftype_integration_test.go
new file mode 100644
index 000000000..bafbcb78a
--- /dev/null
+++ b/pkg/detectors/swiftype/swiftype_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package swiftype
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestSwiftype_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("SWIFTYPE")
+ inactiveSecret := testSecrets.MustGetField("SWIFTYPE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a swiftype secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Swiftype,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a swiftype secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Swiftype,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Swiftype.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Swiftype.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/swiftype/swiftype_test.go b/pkg/detectors/swiftype/swiftype_test.go
index 95930c642..3715353e8 100644
--- a/pkg/detectors/swiftype/swiftype_test.go
+++ b/pkg/detectors/swiftype/swiftype_test.go
@@ -1,120 +1,91 @@
-//go:build detectors
-// +build detectors
-
package swiftype
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestSwiftype_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SWIFTYPE")
- inactiveSecret := testSecrets.MustGetField("SWIFTYPE_INACTIVE")
+var (
+ validPattern = "hJgyi1_y8ohpS-N2vlXE"
+ invalidPattern = "?Jgyi1_y8ohpS-N2vlXE"
+ keyword = "swiftype"
+)
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
+func TestSwiftype_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a swiftype secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Swiftype,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with keyword swiftype",
+ input: fmt.Sprintf("%s token = '%s'", keyword, validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a swiftype secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Swiftype,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore duplicate",
+ input: fmt.Sprintf("%s token = '%s' | '%s'", keyword, validPattern, validPattern),
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("%s keyword is not close to the real key in the data\n = '%s'", keyword, validPattern),
+ want: []string{},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("%s = '%s'", keyword, invalidPattern),
+ want: []string{},
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Swiftype.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Swiftype.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}