Skip to content

Find, verify, and analyze leaked credentials

Added Pattern test cases for detectors (#3354)

Kashif Khana4cc5f7cc38c
diff --git a/pkg/detectors/abbysale/abbysale_integration_test.go b/pkg/detectors/abbysale/abbysale_integration_test.go
new file mode 100644
index 000000000..665ce0b81
--- /dev/null
+++ b/pkg/detectors/abbysale/abbysale_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package abbysale
+
+import (
+ "context"
+ "fmt"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+)
+
+func TestAbbysale_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ABBYSALE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("ABBYSALE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Abbysale,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Abbysale,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Abbysale,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Abbysale,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/abbysale/abbysale_test.go b/pkg/detectors/abbysale/abbysale_test.go
index 8f4744985..329a3eca0 100644
--- a/pkg/detectors/abbysale/abbysale_test.go
+++ b/pkg/detectors/abbysale/abbysale_test.go
@@ -1,167 +1,85 @@
-//go:build detectors
-// +build detectors
-
package abbysale
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "123abcDEF456ghiJKL789mnoPQR012stuVWX3456"
+ invalidPattern = "123abcDEF456ghiJKL789mnoPQR012stuVWX"
)
-func TestAbbysale_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ABBYSALE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("ABBYSALE_INACTIVE")
+func TestAbbySale_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Abbysale,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Abbysale,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Abbysale,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("abbysale token = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abbysale secret %s within but verified", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Abbysale,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("abbysale token keyword is not close to the real token = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("abbysale = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Abbysale.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
+ return
+ }
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Abbysale.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/abstract/abstract_integration_test.go b/pkg/detectors/abstract/abstract_integration_test.go
new file mode 100644
index 000000000..2ed3939c9
--- /dev/null
+++ b/pkg/detectors/abstract/abstract_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package abstract
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAbstract_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ABSTRACT")
+ inactiveSecret := testSecrets.MustGetField("ABSTRACT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Abstract,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Abstract,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Abstract,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Abstract,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ time.Sleep(900 * time.Millisecond) // avoid rate limiting
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Abstract.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Abstract.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/abstract/abstract_test.go b/pkg/detectors/abstract/abstract_test.go
index 2ed3939c9..647c1e564 100644
--- a/pkg/detectors/abstract/abstract_test.go
+++ b/pkg/detectors/abstract/abstract_test.go
@@ -1,167 +1,85 @@
-//go:build detectors
-// +build detectors
-
package abstract
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAbstract_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ABSTRACT")
- inactiveSecret := testSecrets.MustGetField("ABSTRACT_INACTIVE")
+var (
+ validPattern = "qwerty12345iugt67s7a7sa0akhsxz82"
+ invalidPattern = "zxcvbr12345iugt67s7a7sa0akhsXz820"
+)
+
+func TestAbstract_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Abstract,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("abstract token = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abstract secret %s within but verified", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Abstract,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("abstract token keyword is not close to the real token = '%s'", validPattern),
+ want: nil,
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abstract secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Abstract,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abstract secret %s within but not valid", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Abstract,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("abstract = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- time.Sleep(900 * time.Millisecond) // avoid rate limiting
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Abstract.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Abstract.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/abuseipdb/abuseipdb_integration_test.go b/pkg/detectors/abuseipdb/abuseipdb_integration_test.go
new file mode 100644
index 000000000..bd481f080
--- /dev/null
+++ b/pkg/detectors/abuseipdb/abuseipdb_integration_test.go
@@ -0,0 +1,167 @@
+//go:build detectors
+// +build detectors
+
+package abuseipdb
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAbuseIPDB_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ABUSEIPDB")
+ inactiveSecret := testSecrets.MustGetField("ABUSEIPDB_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AbuseIPDB,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AbuseIPDB,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AbuseIPDB,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AbuseIPDB,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AbuseIPDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AbuseIPDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/abuseipdb/abuseipdb_test.go b/pkg/detectors/abuseipdb/abuseipdb_test.go
index bd481f080..eec2bf0fd 100644
--- a/pkg/detectors/abuseipdb/abuseipdb_test.go
+++ b/pkg/detectors/abuseipdb/abuseipdb_test.go
@@ -1,166 +1,85 @@
-//go:build detectors
-// +build detectors
-
package abuseipdb
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAbuseIPDB_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ABUSEIPDB")
- inactiveSecret := testSecrets.MustGetField("ABUSEIPDB_INACTIVE")
+var (
+ validPattern = "123abcdef456ghijkl789mnopqr012stuvwx3455123abcdef456ghijkl789mnopqr012stuvwx3455"
+ invalidPattern = "123abcdef456Ghijkl789mnopqr012stuvwx3455123abcdef456ghijkl789mnopqr012stuvwX"
+)
+
+func TestAbuseipdb_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within but verified", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AbuseIPDB,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("abuseipdb token = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_AbuseIPDB,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("abuseipdb token keyword is not close to the real token = '%s'", validPattern),
+ want: nil,
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_AbuseIPDB,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a abuseipdb secret %s within", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AbuseIPDB,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("abuseipdb = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AbuseIPDB.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AbuseIPDB.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/accuweather/accuweather_integration_test.go b/pkg/detectors/accuweather/accuweather_integration_test.go
new file mode 100644
index 000000000..8dc6f8421
--- /dev/null
+++ b/pkg/detectors/accuweather/accuweather_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package accuweather
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAccuweather_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ACCUWEATHER")
+ inactiveSecret := testSecrets.MustGetField("ACCUWEATHER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Accuweather,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Accuweather,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Accuweather,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Accuweather,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Accuweather.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ got[i].Raw = nil
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Accuweather.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/accuweather/accuweather_test.go b/pkg/detectors/accuweather/accuweather_test.go
index 8dc6f8421..a862f0b0e 100644
--- a/pkg/detectors/accuweather/accuweather_test.go
+++ b/pkg/detectors/accuweather/accuweather_test.go
@@ -1,167 +1,85 @@
-//go:build detectors
-// +build detectors
-
package accuweather
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAccuweather_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ACCUWEATHER")
- inactiveSecret := testSecrets.MustGetField("ACCUWEATHER_INACTIVE")
+var (
+ validPattern = "dqftwc490oPc%xae67sBSF741M56%sd091a"
+ invalidPattern = "dqftwc490oPc%xae67sBSF741M56=sd091a"
+)
+
+func TestAccuWeather_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Accuweather,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("accuweather token = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Accuweather,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("accuweather token keyword is not close to the real token = '%s'", validPattern),
+ want: nil,
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Accuweather,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a accuweather secret %s within but verified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Accuweather,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("accuweather = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Accuweather.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
- got[i].Raw = nil
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Accuweather.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/adobeio/adobeio_integration_test.go b/pkg/detectors/adobeio/adobeio_integration_test.go
new file mode 100644
index 000000000..e1e3914b1
--- /dev/null
+++ b/pkg/detectors/adobeio/adobeio_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package adobeio
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAdobeIO_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ADOBEIO_TOKEN")
+ id := testSecrets.MustGetField("ADOBEIO_PRODUCT")
+ inactiveSecret := testSecrets.MustGetField("ADOBEIO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AdobeIO,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AdobeIO,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AdobeIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AdobeIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/adobeio/adobeio_test.go b/pkg/detectors/adobeio/adobeio_test.go
index a65d0035f..649096da0 100644
--- a/pkg/detectors/adobeio/adobeio_test.go
+++ b/pkg/detectors/adobeio/adobeio_test.go
@@ -1,121 +1,86 @@
-//go:build detectors
-// +build detectors
-
package adobeio
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "zxcv0987mnbv1234poiu6749gtnrfv54/WDcv0981Mn.B"
+ invalidPattern = "Rzxc#0987$%bv1234poiu6749gtnrfv54"
)
-func TestAdobeIO_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ADOBEIO_TOKEN")
- id := testSecrets.MustGetField("ADOBEIO_PRODUCT")
- inactiveSecret := testSecrets.MustGetField("ADOBEIO_INACTIVE")
+func TestAdobeIO_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AdobeIO,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("adobe = '%s'", validPattern),
+ want: []string{"zxcv0987mnbv1234poiu6749gtnrfv54WDcv0981Mn.B"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a adobeio secret %s within adobeio %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AdobeIO,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("adobe keyword is not close to the real id and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("adobe%s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AdobeIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AdobeIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/adzuna/adzuna_integration_test.go b/pkg/detectors/adzuna/adzuna_integration_test.go
new file mode 100644
index 000000000..f2a1bf630
--- /dev/null
+++ b/pkg/detectors/adzuna/adzuna_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package adzuna
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAdzuna_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ADZUNA")
+ id := testSecrets.MustGetField("ADZUNA_ID")
+ inactiveSecret := testSecrets.MustGetField("ADZUNA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Adzuna,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Adzuna,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Adzuna,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Adzuna,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Adzuna.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Adzuna.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/adzuna/adzuna_test.go b/pkg/detectors/adzuna/adzuna_test.go
index f2a1bf630..83dd82037 100644
--- a/pkg/detectors/adzuna/adzuna_test.go
+++ b/pkg/detectors/adzuna/adzuna_test.go
@@ -1,167 +1,96 @@
-//go:build detectors
-// +build detectors
-
package adzuna
import (
"context"
"fmt"
+ "strings"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAdzuna_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ADZUNA")
- id := testSecrets.MustGetField("ADZUNA_ID")
- inactiveSecret := testSecrets.MustGetField("ADZUNA_INACTIVE")
+var (
+ validPattern = "asdf0987mnbv1234qsxojb6ygb2wsx0o/qsc0f098"
+ invalidPattern = "as#f0987mnbv1234^sxojb6ygb2wsx0o/qsc0f098"
+)
+
+func TestAdzuna_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Adzuna,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("adzuna = '%s'", validPattern),
+ want: []string{"asdf0987mnbv1234qsxojb6ygb2wsx0oqsc0f098"},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Adzuna,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("adzuna keyword is not close to the real id and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Adzuna,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - only key",
+ input: fmt.Sprintf("adzuna %s", strings.Split(validPattern, "/")[0]),
+ want: nil,
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a adzuna secret %s within adzuna %s but not valid", inactiveSecret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Adzuna,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - only id",
+ input: fmt.Sprintf("adzuna %s", strings.Split(validPattern, "/")[1]),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("adzuna %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Adzuna.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Adzuna.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/aeroworkflow/aeroworkflow_integration_test.go b/pkg/detectors/aeroworkflow/aeroworkflow_integration_test.go
new file mode 100644
index 000000000..3ea3ef555
--- /dev/null
+++ b/pkg/detectors/aeroworkflow/aeroworkflow_integration_test.go
@@ -0,0 +1,165 @@
+//go:build detectors
+// +build detectors
+
+package aeroworkflow
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAeroworkflow_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AEROWORKFLOW_SECRET")
+ id := testSecrets.MustGetField("AEROWORKFLOW_ID")
+ inactiveSecret := testSecrets.MustGetField("AEROWORKFLOW_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aeroworkflow,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Aeroworkflow,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Aeroworkflow,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aeroworkflow,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Aeroworkflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Aeroworkflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aeroworkflow/aeroworkflow_test.go b/pkg/detectors/aeroworkflow/aeroworkflow_test.go
index ba46cd93e..2d5dc1a08 100644
--- a/pkg/detectors/aeroworkflow/aeroworkflow_test.go
+++ b/pkg/detectors/aeroworkflow/aeroworkflow_test.go
@@ -1,163 +1,96 @@
-//go:build detectors
-// +build detectors
-
package aeroworkflow
import (
"context"
"fmt"
+ "strings"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAeroworkflow_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AEROWORKFLOW_SECRET")
- id := testSecrets.MustGetField("AEROWORKFLOW_ID")
- inactiveSecret := testSecrets.MustGetField("AEROWORKFLOW_INACTIVE")
+var (
+ validPattern = "qscVgy!WdvG;^#O:*?TG/806445634"
+ invalidPattern = "qscVg&!WdvG;^#O:*?TG/8064456A4"
+)
+
+func TestAeroWorkflow_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aeroworkflow,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("aeroworkflow = '%s'", validPattern),
+ want: []string{"qscVgy!WdvG;^#O:*?TG806445634"},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Aeroworkflow,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("aeroworkflow keyword is not close to the real id and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Aeroworkflow,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - only key",
+ input: fmt.Sprintf("aeroworkflow %s", strings.Split(validPattern, "/")[0]),
+ want: nil,
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aeroworkflow secret %s within aeroworkflow %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aeroworkflow,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - only id",
+ input: fmt.Sprintf("aeroworkflow %s", strings.Split(validPattern, "/")[1]),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("aeroworkflow %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Aeroworkflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Aeroworkflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/agora/agora_integration_test.go b/pkg/detectors/agora/agora_integration_test.go
new file mode 100644
index 000000000..fbd70bd86
--- /dev/null
+++ b/pkg/detectors/agora/agora_integration_test.go
@@ -0,0 +1,192 @@
+//go:build detectors
+// +build detectors
+
+package agora
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAgora_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("AGORA")
+ secret := testSecrets.MustGetField("AGORA_SECRET")
+ inactiveSecret := testSecrets.MustGetField("AGORA_SECRET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: true,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r, r, r, r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r, r, r, r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but not valid ", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ {
+ DetectorType: detectorspb.DetectorType_Agora,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/agora/agora_test.go b/pkg/detectors/agora/agora_test.go
index db0f64ac9..4a16637fe 100644
--- a/pkg/detectors/agora/agora_test.go
+++ b/pkg/detectors/agora/agora_test.go
@@ -1,191 +1,96 @@
-//go:build detectors
-// +build detectors
-
package agora
import (
"context"
"fmt"
+ "strings"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "asdf0987mnbv1234qsxojb6ygb2wsx0o/beqr7215fr4g6bfjkmnvxrtygb2wsxap"
+ invalidPattern = "asdf0987mNbv1234qsxojb6ygb2w$x0o/beqr7215fr4g6bfjkmnVxrtygb2wsxap"
)
-func TestAgora_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("AGORA")
- secret := testSecrets.MustGetField("AGORA_SECRET")
- inactiveSecret := testSecrets.MustGetField("AGORA_SECRET_INACTIVE")
+func TestAgora_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: true,
- },
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("agora = '%s'", validPattern),
+ want: []string{"asdf0987mnbv1234qsxojb6ygb2wsx0oasdf0987mnbv1234qsxojb6ygb2wsx0o"},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r, r, r, r}
- }(),
- wantErr: false,
+ name: "valid pattern - out of prefix range",
+ input: fmt.Sprintf("agora keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but verified", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r, r, r, r}
- }(),
- wantErr: false,
+ name: "valid pattern - only key",
+ input: fmt.Sprintf("agora %s", strings.Split(validPattern, "/")[0]),
+ want: []string{"asdf0987mnbv1234qsxojb6ygb2wsx0oasdf0987mnbv1234qsxojb6ygb2wsx0o"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a agora secret %s within agora id %s but not valid ", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- {
- DetectorType: detectorspb.DetectorType_Agora,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - only secret",
+ input: fmt.Sprintf("agora %s", strings.Split(validPattern, "/")[1]),
+ want: []string{"beqr7215fr4g6bfjkmnvxrtygb2wsxapbeqr7215fr4g6bfjkmnvxrtygb2wsxap"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("agora %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Agora.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Agora.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/aha/aha_integration_test.go b/pkg/detectors/aha/aha_integration_test.go
new file mode 100644
index 000000000..548beb044
--- /dev/null
+++ b/pkg/detectors/aha/aha_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package aha
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAha_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ domain := testSecrets.MustGetField("AHA_DOMAIN")
+ secret := testSecrets.MustGetField("AHA_SECRET")
+ inactiveSecret := testSecrets.MustGetField("AHA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{client: common.ConstantResponseHttpClient(200, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aha,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Aha,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Aha,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid domain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aha,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Aha.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Aha.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aha/aha_test.go b/pkg/detectors/aha/aha_test.go
index 5e3a93784..d68de04b7 100644
--- a/pkg/detectors/aha/aha_test.go
+++ b/pkg/detectors/aha/aha_test.go
@@ -1,167 +1,104 @@
-//go:build detectors
-// +build detectors
-
package aha
import (
"context"
"fmt"
+ "strings"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff/example.aha.io"
+ invalidPattern = "00112233445566778899aabbCC$%eeff00112233445566778899aabbccddeeff/example.fake.io"
)
-func TestAha_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- domain := testSecrets.MustGetField("AHA_DOMAIN")
- secret := testSecrets.MustGetField("AHA_SECRET")
- inactiveSecret := testSecrets.MustGetField("AHA_INACTIVE")
+func TestAha_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
+ key := strings.Split(validPattern, "/")[0]
+ url := strings.Split(validPattern, "/")[1]
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{client: common.ConstantResponseHttpClient(200, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aha,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("aha.io = '%s'", validPattern),
+ want: []string{key},
},
{
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Aha,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - detect URL far away from keyword",
+ input: fmt.Sprintf("aha.io = '%s\n URL is not close to the keyword but should be detected %s'", key, url),
+ want: []string{key},
},
{
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aha secret %s within %s but verified", secret, domain)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Aha,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("aha.io keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
+ },
+ {
+ name: "valid pattern - only key",
+ input: fmt.Sprintf("aha.io %s", key),
+ want: []string{key},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aha secret %s within but not valid domain %s", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aha,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - only URL",
+ input: fmt.Sprintf("aha.io %s", url),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("aha.io %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Aha.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Aha.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_integration_test.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_integration_test.go
new file mode 100644
index 000000000..1dc9cc5ec
--- /dev/null
+++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package airbrakeprojectkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAirbrakeProjectKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_TOKEN")
+ id := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_ID")
+ inactiveSecret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but verified", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AirbrakeProjectKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AirbrakeProjectKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go
index 3ce173571..7c70bb9c3 100644
--- a/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go
+++ b/pkg/detectors/airbrakeprojectkey/airbrakeprojectkey_test.go
@@ -1,121 +1,97 @@
-//go:build detectors
-// +build detectors
-
package airbrakeprojectkey
import (
"context"
"fmt"
+ "strings"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "qwmnerBv56zxpocvkjqr78afvYUx90Op/451298"
+ invalidPattern = "qwmnerBv56zxpocvkjqr78afvYU$90Op/4512987"
)
-func TestAirbrakeProjectKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_TOKEN")
- id := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_ID")
- inactiveSecret := testSecrets.MustGetField("AIRBRAKEPROJECTKEY_INACTIVE")
+func TestAirBrakeProjectKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but verified", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("airbrake = '%s'", validPattern),
+ want: []string{"qwmnerBv56zxpocvkjqr78afvYUx90Op451298"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airbrake secret %s within airbrake %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("airbrake keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - only key",
+ input: fmt.Sprintf("airbrake %s", strings.Split(validPattern, "/")[0]),
+ want: nil,
+ },
+ {
+ name: "valid pattern - only ID",
+ input: fmt.Sprintf("airbrake %s", strings.Split(validPattern, "/")[0]),
+ want: nil,
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("airbrake = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AirbrakeProjectKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AirbrakeProjectKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_integration_test.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_integration_test.go
new file mode 100644
index 000000000..7e0f5fabc
--- /dev/null
+++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package airbrakeuserkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAirbrakeUserKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AIRBRAKEUSERKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("AIRBRAKEUSERKEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirbrakeUserKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirbrakeUserKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AirbrakeUserKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AirbrakeUserKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go
index 462603268..b41696885 100644
--- a/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go
+++ b/pkg/detectors/airbrakeuserkey/airbrakeuserkey_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package airbrakeuserkey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "qsCGuilpkk2ngrsz75wtYqsCGuilpkk2ngrsz75w"
+ invalidPattern = "Qs%CGuil#pkk2ngrsz75wtYqsCGuilpkk2ngrsz75w"
)
-func TestAirbrakeUserKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AIRBRAKEUSERKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("AIRBRAKEUSERKEY_INACTIVE")
+func TestAirBrakeUserKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirbrakeUserKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("airbrake = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airbrakeuserkey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirbrakeUserKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("airbrake keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("airbrake = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AirbrakeUserKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AirbrakeUserKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/airship/airship.go b/pkg/detectors/airship/airship.go
index 5c543df08..57eabd568 100644
--- a/pkg/detectors/airship/airship.go
+++ b/pkg/detectors/airship/airship.go
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"airship"}) + `\b([0-9Aa-zA-Z]{91})\b`)
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"airship"}) + `\b([0-9a-zA-Z]{91})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
diff --git a/pkg/detectors/airship/airship_integration_test.go b/pkg/detectors/airship/airship_integration_test.go
new file mode 100644
index 000000000..8969b280b
--- /dev/null
+++ b/pkg/detectors/airship/airship_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package airship
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAirship_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AIRSHIP")
+ inactiveSecret := testSecrets.MustGetField("AIRSHIP_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airship secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Airship,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airship secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Airship,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Airship.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Airship.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/airship/airship_test.go b/pkg/detectors/airship/airship_test.go
index 75f3263db..dc354533b 100644
--- a/pkg/detectors/airship/airship_test.go
+++ b/pkg/detectors/airship/airship_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package airship
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "123abcdef456ghijkl789mnopqr012stuvwx3455123abcdef456ghijkl789mnopqr012stuvwx3455trdfrq95acr"
+ invalidPattern = "123#abcdef456gh$ijkl789mnopqr012stuvwX3455123abcdef456ghijkl789mnopqr012stuvwx3455trdfrq"
)
-func TestAirship_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AIRSHIP")
- inactiveSecret := testSecrets.MustGetField("AIRSHIP_INACTIVE")
+func TestAirship_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airship secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Airship,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("airship = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airship secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Airship,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("airship keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("airship = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Airship.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Airship.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/airtableapikey/airtableapikey.go b/pkg/detectors/airtableapikey/airtableapikey.go
index 0938bba0b..bff9284d8 100644
--- a/pkg/detectors/airtableapikey/airtableapikey.go
+++ b/pkg/detectors/airtableapikey/airtableapikey.go
@@ -13,7 +13,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@@ -51,7 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if len(keyMatches) == 0 {
keyMatches = personalKeyMatches
-
}
for _, keyMatch := range keyMatches {
diff --git a/pkg/detectors/airtableapikey/airtableapikey_integration_test.go b/pkg/detectors/airtableapikey/airtableapikey_integration_test.go
new file mode 100644
index 000000000..ba75a3864
--- /dev/null
+++ b/pkg/detectors/airtableapikey/airtableapikey_integration_test.go
@@ -0,0 +1,125 @@
+//go:build detectors
+// +build detectors
+
+package airtableapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAirtableApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY")
+ inactiveSecret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY_INACTIVE")
+ app := testSecrets.MustGetField("AIRTABLEAPIKEY_APP")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable https://api.airtable.com/v0/%s/Projects", secret, app)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirtableApiKey,
+ Redacted: app,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable %s", inactiveSecret, app)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirtableApiKey,
+ Redacted: app,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AirtableApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/airtableapikey/airtableapikey_test.go b/pkg/detectors/airtableapikey/airtableapikey_test.go
index 6c3e962ed..a9ddfa89a 100644
--- a/pkg/detectors/airtableapikey/airtableapikey_test.go
+++ b/pkg/detectors/airtableapikey/airtableapikey_test.go
@@ -1,125 +1,87 @@
-//go:build detectors
-// +build detectors
-
package airtableapikey
import (
"context"
"fmt"
"testing"
- "time"
+
+ "github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "app_pOcv67-Yuztyq / key_Yuztyq-pOcv67"
+ validPersonalKeyPattern = "app_pOcv67-Yuztyq / patWtrafrcg64DP0w.AA1AA2BB3CC4DD5EE6FF7GG8HH9II0JJ1KK2LL3MM4NN5OO6PP7QQ8aaaaaaaaaa"
+ invalidPattern = "app_pOcv67%Yuztyq/key_Yuztyq*pOcv67"
)
-func TestAirtableApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY")
- inactiveSecret := testSecrets.MustGetField("AIRTABLEAPIKEY_KEY_INACTIVE")
- app := testSecrets.MustGetField("AIRTABLEAPIKEY_APP")
+func TestAirTableApiKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable https://api.airtable.com/v0/%s/Projects", secret, app)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirtableApiKey,
- Redacted: app,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern - with key",
+ input: fmt.Sprintf("airtable secrets: %s", validPattern),
+ want: []string{"key_Yuztyq-pOcv67app_pOcv67-Yuztyq"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airtableapikey secret %s within airtable %s", inactiveSecret, app)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirtableApiKey,
- Redacted: app,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with personal key",
+ input: fmt.Sprintf("airtable secrets: %s", validPersonalKeyPattern),
+ want: []string{"patWtrafrcg64DP0w.AA1AA2BB3CC4DD5EE6FF7GG8HH9II0JJ1KK2LL3MM4NN5OO6PP7QQ8aaaaaaaaaaapp_pOcv67-Yuztyq"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("airtable secrets: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AirtableApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Airtable.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/airvisual/airvisual_integration_test.go b/pkg/detectors/airvisual/airvisual_integration_test.go
new file mode 100644
index 000000000..74af10ded
--- /dev/null
+++ b/pkg/detectors/airvisual/airvisual_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package airvisual
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAirVisual_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AIRVISUAL")
+ inactiveSecret := testSecrets.MustGetField("AIRVISUAL_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airvisual secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirVisual,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a airvisual secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AirVisual,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AirVisual.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AirVisual.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/airvisual/airvisual_test.go b/pkg/detectors/airvisual/airvisual_test.go
index cef7e9488..b4d585384 100644
--- a/pkg/detectors/airvisual/airvisual_test.go
+++ b/pkg/detectors/airvisual/airvisual_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package airvisual
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "qscgyygcsq-wdvvok7slklklaasnd8afafxd"
+ invalidPattern = "wdvvok7slklklaasnd8afafxd"
)
-func TestAirVisual_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AIRVISUAL")
- inactiveSecret := testSecrets.MustGetField("AIRVISUAL_INACTIVE")
+func TestAirVisual_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airvisual secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirVisual,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("airvisual = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a airvisual secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AirVisual,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("airvisual keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("airvisual key: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AirVisual.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AirVisual.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/aiven/aiven_integration_test.go b/pkg/detectors/aiven/aiven_integration_test.go
new file mode 100644
index 000000000..5939a1726
--- /dev/null
+++ b/pkg/detectors/aiven/aiven_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package aiven
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAiven_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AIVEN")
+ inactiveSecret := testSecrets.MustGetField("AIVEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aiven secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aiven,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aiven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aiven,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Aiven.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Aiven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aiven/aiven_test.go b/pkg/detectors/aiven/aiven_test.go
index 69ba404ca..f6447a02f 100644
--- a/pkg/detectors/aiven/aiven_test.go
+++ b/pkg/detectors/aiven/aiven_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package aiven
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef123/+==ABCdef12`
+ invalidPattern = `ABCdef123`
)
-func TestAiven_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AIVEN")
- inactiveSecret := testSecrets.MustGetField("AIVEN_INACTIVE")
+func TestAiven_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aiven secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aiven,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("aiven = '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aiven secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aiven,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("aiven keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("aiven = '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Aiven.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Aiven.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/alconost/alconost_integration_test.go b/pkg/detectors/alconost/alconost_integration_test.go
new file mode 100644
index 000000000..6e3fda8fd
--- /dev/null
+++ b/pkg/detectors/alconost/alconost_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package alconost
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAlconost_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALCONOST")
+ inactiveSecret := testSecrets.MustGetField("ALCONOST_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alconost secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alconost,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alconost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alconost,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Alconost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Alconost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/alconost/alconost_test.go b/pkg/detectors/alconost/alconost_test.go
index 96551e827..0c4cc1f0e 100644
--- a/pkg/detectors/alconost/alconost_test.go
+++ b/pkg/detectors/alconost/alconost_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package alconost
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "wdvnousa87acfxp9ioasrea4tbeasrfa"
+ invalidPattern = "wdvnousa87acfxp9ioasr$a4tBeasrfa"
)
-func TestAlconost_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALCONOST")
- inactiveSecret := testSecrets.MustGetField("ALCONOST_INACTIVE")
+func TestAlconost_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alconost secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Alconost,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("alconost: '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alconost secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Alconost,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("alconost keyword is not close to the real key in the data = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("alconost: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Alconost.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Alconost.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/alegra/alegra.go b/pkg/detectors/alegra/alegra.go
index 065b44233..1fcbfcb2b 100644
--- a/pkg/detectors/alegra/alegra.go
+++ b/pkg/detectors/alegra/alegra.go
@@ -12,7 +12,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@@ -24,7 +24,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-z0-9-]{20})\b`)
- idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9.-@]{25,30})\b`)
+ idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alegra"}) + `\b([a-zA-Z0-9\.\-\@]{25,30})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
diff --git a/pkg/detectors/alegra/alegra_integration_test.go b/pkg/detectors/alegra/alegra_integration_test.go
new file mode 100644
index 000000000..19913d9c8
--- /dev/null
+++ b/pkg/detectors/alegra/alegra_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package alegra
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAlegra_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALEGRA")
+ id := testSecrets.MustGetField("ACCOUNT_USER")
+ inactiveSecret := testSecrets.MustGetField("ALEGRA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alegra,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alegra,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Alegra.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/alegra/alegra_test.go b/pkg/detectors/alegra/alegra_test.go
index 31fe6fb58..d2eb6e346 100644
--- a/pkg/detectors/alegra/alegra_test.go
+++ b/pkg/detectors/alegra/alegra_test.go
@@ -1,121 +1,92 @@
-//go:build detectors
-// +build detectors
-
package alegra
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "wdvn-usa87a-fxp9ioas/[email protected]"
+ validSpecialCharPattern = "wdvn-usa87a-fxp9ioas / [email protected]"
+ invalidPattern = "wdvn-usa87a-fxp9ioas/[email protected]"
)
-func TestAlegra_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALEGRA")
- id := testSecrets.MustGetField("ACCOUNT_USER")
- inactiveSecret := testSecrets.MustGetField("ALEGRA_INACTIVE")
+func TestAlegra_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Alegra,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("alegra: '%s'", validPattern),
+ want: []string{"[email protected]"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alegra secret %s within alegra %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Alegra,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with special characters",
+ input: fmt.Sprintf("alegra: '%s'", validSpecialCharPattern),
+ want: []string{"[email protected]"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("alegra keyword is not close to the real key and id = '%s'", validPattern),
+ want: nil,
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("alegra: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Alegra.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Alegra.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/aletheiaapi/aletheiaapi_integration_test.go b/pkg/detectors/aletheiaapi/aletheiaapi_integration_test.go
new file mode 100644
index 000000000..32b39a124
--- /dev/null
+++ b/pkg/detectors/aletheiaapi/aletheiaapi_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package aletheiaapi
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAletheiaApi_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALETHEIAAPI")
+ inactiveSecret := testSecrets.MustGetField("ALETHEIAAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AletheiaApi,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AletheiaApi,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AletheiaApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AletheiaApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aletheiaapi/aletheiaapi_test.go b/pkg/detectors/aletheiaapi/aletheiaapi_test.go
index 32b39a124..e651d037f 100644
--- a/pkg/detectors/aletheiaapi/aletheiaapi_test.go
+++ b/pkg/detectors/aletheiaapi/aletheiaapi_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package aletheiaapi
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "TEST01KEYPATTERN2024ALETHEIAAPIN"
+ invalidPattern = "TEST01kEYPATTERN2024ALETHEIAAPIN"
)
-func TestAletheiaApi_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALETHEIAAPI")
- inactiveSecret := testSecrets.MustGetField("ALETHEIAAPI_INACTIVE")
+func TestAleTheIaAPI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AletheiaApi,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("aletheiaapi: '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aletheiaapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AletheiaApi,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("aletheiaapi keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("aletheiaapi: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AletheiaApi.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AletheiaApi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go
new file mode 100644
index 000000000..3a6985580
--- /dev/null
+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package algoliaadminkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAlgoliaAdminKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALGOLIAADMINKEY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("ALGOLIAADMINKEY_INACTIVE")
+ id := testSecrets.MustGetField("ALGOLIAADMINKEY_APPID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AlgoliaAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AlgoliaAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go
index 3a6985580..a433d34a7 100644
--- a/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go
+++ b/pkg/detectors/algoliaadminkey/algoliaadminkey_test.go
@@ -1,121 +1,86 @@
-//go:build detectors
-// +build detectors
-
package algoliaadminkey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "AlgoliaAdminKey010TESTINGPlmiOPC/ADMIN01KEY"
+ invalidPattern = "AlgoliaAdminN$TKey010TESTINGPlmiOPC/NonAdmin01ID"
)
-func TestAlgoliaAdminKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALGOLIAADMINKEY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("ALGOLIAADMINKEY_INACTIVE")
- id := testSecrets.MustGetField("ALGOLIAADMINKEY_APPID")
+func TestAlgoliaAdminKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("algolia: '%s'", validPattern),
+ want: []string{"AlgoliaAdminKey010TESTINGPlmiOPCADMIN01KEY"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a algolia secret %s within algolia %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("algolia keyword is not close to the real key and secret = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("algolia: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AlgoliaAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AlgoliaAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/alibaba/alibaba_integration_test.go b/pkg/detectors/alibaba/alibaba_integration_test.go
new file mode 100644
index 000000000..31b81881e
--- /dev/null
+++ b/pkg/detectors/alibaba/alibaba_integration_test.go
@@ -0,0 +1,186 @@
+//go:build detectors
+// +build detectors
+
+package alibaba
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAlibaba_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALIBABA_SECRET")
+ inactiveSecret := testSecrets.MustGetField("ALIBABA_SECRET_INACTIVE")
+ id := testSecrets.MustGetField("ALIBABA_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to broken json",
+ s: Scanner{client: common.ConstantResponseHttpClient(418, "{")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected EOF"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Alibaba,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/alibaba/alibaba_test.go b/pkg/detectors/alibaba/alibaba_test.go
index 31b81881e..8edb6f4cc 100644
--- a/pkg/detectors/alibaba/alibaba_test.go
+++ b/pkg/detectors/alibaba/alibaba_test.go
@@ -1,185 +1,85 @@
-//go:build detectors
-// +build detectors
-
package alibaba
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "abcDEF123ghiJKL456mnoPQR789std/LTAI123ABCdef456ghijkLMN"
+ invalidPattern = "abcDEF123ghiJKL456m$oPQR789/123ABCdef456ghijkLMN;"
)
-func TestAlibaba_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALIBABA_SECRET")
- inactiveSecret := testSecrets.MustGetField("ALIBABA_SECRET_INACTIVE")
- id := testSecrets.MustGetField("ALIBABA_ID")
+func TestAliBaba_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Alibaba,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Alibaba,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Alibaba,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to broken json",
- s: Scanner{client: common.ConstantResponseHttpClient(418, "{")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s", secret, id)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_Alibaba,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected EOF"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("LTAI: '%s'", validPattern),
+ want: []string{"abcDEF123ghiJKL456mnoPQR789stdLTAI123ABCdef456ghijkLMN"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alibaba secret %s within alibaba %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Alibaba,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - ignore special characters at end",
+ input: fmt.Sprintf("LTAI: '%s\"''", validPattern),
+ want: []string{"abcDEF123ghiJKL456mnoPQR789stdLTAI123ABCdef456ghijkLMN"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("LTAI: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Alibaba.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Alibaba.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/alienvault/alienvault_integration_test.go b/pkg/detectors/alienvault/alienvault_integration_test.go
new file mode 100644
index 000000000..ab9e035bd
--- /dev/null
+++ b/pkg/detectors/alienvault/alienvault_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package alienvault
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAlienVault_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALIENVAULT")
+ inactiveSecret := testSecrets.MustGetField("ALIENVAULT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alienvault secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AlienVault,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a alienvault secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AlienVault,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AlienVault.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AlienVault.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/alienvault/alienvault_test.go b/pkg/detectors/alienvault/alienvault_test.go
index afff2582a..514c5de34 100644
--- a/pkg/detectors/alienvault/alienvault_test.go
+++ b/pkg/detectors/alienvault/alienvault_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package alienvault
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "a3b9f8c1e2d4f5b6c7d8e9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0"
+ invalidPattern = "A$3b9f8c1e2d4f5b6c7d8e9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"
)
-func TestAlienVault_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALIENVAULT")
- inactiveSecret := testSecrets.MustGetField("ALIENVAULT_INACTIVE")
+func TestAlienVault_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alienvault secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AlienVault,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("alienvault: '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a alienvault secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AlienVault,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("alienvault keyword is not close to the real key in the data\n = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("alienvault: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AlienVault.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AlienVault.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}