Skip to content

Find, verify, and analyze leaked credentials

added pattern test cases for all detectors starting with Alphabet `a` (#3539)

Kashif Khan8897ed2a66fa
diff --git a/pkg/detectors/allsports/allsports_integration_test.go b/pkg/detectors/allsports/allsports_integration_test.go
new file mode 100644
index 000000000..0619c62f7
--- /dev/null
+++ b/pkg/detectors/allsports/allsports_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package allsports
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAllsports_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ALLSPORTS")
+ inactiveSecret := testSecrets.MustGetField("ALLSPORTS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a allsports secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Allsports,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a allsports secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Allsports,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Allsports.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Allsports.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/allsports/allsports_test.go b/pkg/detectors/allsports/allsports_test.go
index edf6454fc..81414fd8f 100644
--- a/pkg/detectors/allsports/allsports_test.go
+++ b/pkg/detectors/allsports/allsports_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package allsports
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "d1f2e3c4b5a6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1s2t3u4v5w6x7y8z9a0b1ce"
+ invalidPattern = "Ad1f2e3c4b5a6d7e8f9g0h1i2j3k4l5m6n7o8p9q0r1sRt3u4v5w6x7y8z9a0b1cE"
)
-func TestAllsports_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ALLSPORTS")
- inactiveSecret := testSecrets.MustGetField("ALLSPORTS_INACTIVE")
+func TestAllSports_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a allsports secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Allsports,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("allsports: '%s'", validPattern),
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a allsports secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Allsports,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("allsports keyword is not close to the real key in the data\n = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("allsports: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Allsports.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Allsports.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/amadeus/amadeus_integration_test.go b/pkg/detectors/amadeus/amadeus_integration_test.go
new file mode 100644
index 000000000..0f87057ca
--- /dev/null
+++ b/pkg/detectors/amadeus/amadeus_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package amadeus
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAmadeus_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("AMADEUS")
+ secret := testSecrets.MustGetField("AMADEUS_SECRET")
+ inactiveSecret := testSecrets.MustGetField("AMADEUS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Amadeus,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Amadeus,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Amadeus.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Amadeus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/amadeus/amadeus_test.go b/pkg/detectors/amadeus/amadeus_test.go
index 0f87057ca..5437f5f06 100644
--- a/pkg/detectors/amadeus/amadeus_test.go
+++ b/pkg/detectors/amadeus/amadeus_test.go
@@ -1,121 +1,86 @@
-//go:build detectors
-// +build detectors
-
package amadeus
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6 / A1b2C3d4E5f6G7ho"
+ invalidPattern = "A1b2$3d4E5f6G7h8I9j(K1l2M3n4O5)6 / A1b2C3d4E^f6G7#8"
)
-func TestAmadeus_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("AMADEUS")
- secret := testSecrets.MustGetField("AMADEUS_SECRET")
- inactiveSecret := testSecrets.MustGetField("AMADEUS_INACTIVE")
+func TestAmadeus_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Amadeus,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("amadeus: '%s'", validPattern),
+ want: []string{"A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6A1b2C3d4E5f6G7ho"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a amadeus secret %s within amadeus id %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Amadeus,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("amadeus keyword is not close to the real key in the data\n = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("amadeus: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Amadeus.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Amadeus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/ambee/ambee_integration_test.go b/pkg/detectors/ambee/ambee_integration_test.go
new file mode 100644
index 000000000..6ff21de9d
--- /dev/null
+++ b/pkg/detectors/ambee/ambee_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package ambee
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAmbee_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AMBEE")
+ inactiveSecret := testSecrets.MustGetField("AMBEE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ambee secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ambee,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ambee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ambee,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Ambee.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Ambee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/ambee/ambee_test.go b/pkg/detectors/ambee/ambee_test.go
index 6ff21de9d..12dc72b45 100644
--- a/pkg/detectors/ambee/ambee_test.go
+++ b/pkg/detectors/ambee/ambee_test.go
@@ -1,120 +1,86 @@
-//go:build detectors
-// +build detectors
-
package ambee
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4"
+ invalidPattern = "X3b4c5d6e7f8a9b0c1d2e3f4a5b6c7Z8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4"
)
-func TestAmbee_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AMBEE")
- inactiveSecret := testSecrets.MustGetField("AMBEE_INACTIVE")
+func TestAmbee_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ambee secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ambee,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("ambee: '%s'", validPattern),
+ want: []string{"a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ambee secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ambee,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - key out of prefix range",
+ input: fmt.Sprintf("ambee keyword is not close to the real key in the data\n = '%s'", validPattern),
+ want: nil,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("ambee: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Ambee.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Ambee.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey.go b/pkg/detectors/amplitudeapikey/amplitudeapikey.go
index 7f5a3d6e3..6e041d760 100644
--- a/pkg/detectors/amplitudeapikey/amplitudeapikey.go
+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey.go
@@ -2,16 +2,17 @@ package amplitudeapikey
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@@ -51,6 +52,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
resSecretMatch := strings.TrimSpace(secretMatch[1])
+ // regex for both key and secret are same so the set of strings could possibly be same as well
+ if resMatch == resSecretMatch {
+ continue
+ }
+
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
Raw: []byte(resMatch),
diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go
new file mode 100644
index 000000000..9028ed7fb
--- /dev/null
+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_integration_test.go
@@ -0,0 +1,124 @@
+//go:build detectors
+// +build detectors
+
+package amplitudeapikey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAmplitudeApiKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+
+ key := testSecrets.MustGetField("AMPLITUDEAPI_KEY")
+ secret := testSecrets.MustGetField("AMPLITUDEAPI_SECRET")
+ inactiveKey := testSecrets.MustGetField("AMPLITUDEAPI_KEY_INACTIVE")
+ inactiveSecret := testSecrets.MustGetField("AMPLITUDEAPI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within", key, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+
+ data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AmplitudeApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AmplitudeApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go
index 9028ed7fb..e9fb9c351 100644
--- a/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go
+++ b/pkg/detectors/amplitudeapikey/amplitudeapikey_test.go
@@ -1,124 +1,90 @@
-//go:build detectors
-// +build detectors
-
package amplitudeapikey
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAmplitudeApiKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
+var (
+ validPattern = `
+ amplitude key = 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d
+ amplitude secret = 5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e
+ `
+ invalidPattern = `
+ amplitude key = 1a2b3c4d5e6f7a8g9c0d1e2f3a4b5c6d
+ amplitude secret = 5b3c4d5e5f7a8b9c0d1r2f2f3a4b4c6e
+ `
+)
- key := testSecrets.MustGetField("AMPLITUDEAPI_KEY")
- secret := testSecrets.MustGetField("AMPLITUDEAPI_SECRET")
- inactiveKey := testSecrets.MustGetField("AMPLITUDEAPI_KEY_INACTIVE")
- inactiveSecret := testSecrets.MustGetField("AMPLITUDEAPI_INACTIVE")
+func TestAmplitudeAPIKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within", key, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
- Verified: true,
- },
+ name: "valid pattern",
+ input: fmt.Sprintf("amplitude: '%s'", validPattern),
+ want: []string{
+ "1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e",
+ "5b3c4d5e5f7a8b9c0d1e2f2f3a4b4c6e1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d",
},
- wantErr: false,
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
-
- data: []byte(fmt.Sprintf("You can find an amplitude key %s with amplitude secret %s within but not valid", inactiveKey, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("amplitude: '%s'", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AmplitudeApiKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AmplitudeApiKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/anthropic/anthropic_integration_test.go b/pkg/detectors/anthropic/anthropic_integration_test.go
new file mode 100644
index 000000000..64713de61
--- /dev/null
+++ b/pkg/detectors/anthropic/anthropic_integration_test.go
@@ -0,0 +1,162 @@
+//go:build detectors
+// +build detectors
+
+package anthropic
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAnthropic_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ANTHROPIC")
+ inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Anthropic,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Anthropic,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Anthropic,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Anthropic,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/anthropic/anthropic_test.go b/pkg/detectors/anthropic/anthropic_test.go
index 64713de61..ba1f085ac 100644
--- a/pkg/detectors/anthropic/anthropic_test.go
+++ b/pkg/detectors/anthropic/anthropic_test.go
@@ -1,161 +1,79 @@
-//go:build detectors
-// +build detectors
-
package anthropic
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "sk-ant-api03-abc123xyz-456def789ghij-klmnopqrstuvwx-3456yza789bcde-1234fghijklmnopby56aaaogaopaaaabc123xyzAA"
+ invalidPattern = "sk-ant-api03-abc123xyz-456de-klMnopqrstuvwx-3456yza789bcde-1234fghijklmnopAA"
)
-func TestAnthropic_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ANTHROPIC")
- inactiveSecret := testSecrets.MustGetField("ANTHROPIC_INACTIVE")
+func TestAnthropic_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Anthropic,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Anthropic,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{validPattern},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Anthropic,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a anthropic secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Anthropic,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Anthropic.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Anthropic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/anypoint/anypoint.go b/pkg/detectors/anypoint/anypoint.go
index c12bcb5bc..d49643ceb 100644
--- a/pkg/detectors/anypoint/anypoint.go
+++ b/pkg/detectors/anypoint/anypoint.go
@@ -3,10 +3,11 @@ package anypoint
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -51,6 +52,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
orgRes := strings.TrimSpace(orgMatch[1])
+ // regex for both key and org are same, so to avoid same string processing
+ if resMatch == orgRes {
+ continue
+ }
+
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Anypoint,
Raw: []byte(resMatch),
diff --git a/pkg/detectors/anypoint/anypoint_integration_test.go b/pkg/detectors/anypoint/anypoint_integration_test.go
new file mode 100644
index 000000000..e07331436
--- /dev/null
+++ b/pkg/detectors/anypoint/anypoint_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package anypoint
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAnypoint_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ANYPOINT_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE")
+ organizationId := testSecrets.MustGetField("ANYPOINT_ORGANIZATIONID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s", secret, organizationId)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Anypoint,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s but not valid", inactiveSecret, organizationId)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Anypoint,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Anypoint.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/anypoint/anypoint_test.go b/pkg/detectors/anypoint/anypoint_test.go
index e07331436..31f5a91df 100644
--- a/pkg/detectors/anypoint/anypoint_test.go
+++ b/pkg/detectors/anypoint/anypoint_test.go
@@ -1,121 +1,107 @@
-//go:build detectors
-// +build detectors
-
package anypoint
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = "1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6 / org: abcd1234-ef56-gh78-ij90-klmn1234opqr"
+ complexPattern = `
+ # Secret Configuration File
+ # Organization details
+ ORG_NAME=my_organization
+ ORG_ID=abcd1234-ef56-gh78-ij90-klmn1234opqr
+
+ # Database credentials
+ DB_USERNAME=iamnotadmin
+ DB_PASSWORD=8f3b6d3e7c9a2f5e
+
+ # OAuth tokens
+ ACCESS_TOKEN=abcxyz123
+ REFRESH_TOKEN=zyxwvutsrqponmlkji9876543210abcd
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ # API keys
+ API_KEY=sk-ant-api03-nothing-just-some-random-api-key-1234fghijklmnopAA
+ SECRET_KEY=1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6
+
+ # Endpoints
+ SERVICE_URL=https://api.example.com/v1/resource
+ `
+ invalidPattern = "1a2b3C4d-5E6f-7g8H-9i0J-k1l2M3n4o5p6 / abcd1234-eF56-gH78-ij90-klmn1234opqr"
)
-func TestAnypoint_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ANYPOINT_TOKEN")
- inactiveSecret := testSecrets.MustGetField("ANYPOINT_INACTIVE")
- organizationId := testSecrets.MustGetField("ANYPOINT_ORGANIZATIONID")
+func TestAnypoint_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s", secret, organizationId)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Anypoint,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("anypoint credentials: %s", validPattern),
+ want: []string{"1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6abcd1234-ef56-gh78-ij90-klmn1234opqr"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find an anypoint secret %s within organization %s but not valid", inactiveSecret, organizationId)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Anypoint,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - complex",
+ input: fmt.Sprintf("anypoint credentials: %s", complexPattern),
+ want: []string{"1a2b3c4d-5e6f-7g8h-9i0j-k1l2m3n4o5p6abcd1234-ef56-gh78-ij90-klmn1234opqr"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("anypoint credentials: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Anypoint.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Anypoint.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apacta/apacta_integration_test.go b/pkg/detectors/apacta/apacta_integration_test.go
new file mode 100644
index 000000000..d80e7f04b
--- /dev/null
+++ b/pkg/detectors/apacta/apacta_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package apacta
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApacta_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APACTA")
+ inactiveSecret := testSecrets.MustGetField("APACTA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apacta secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apacta,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apacta secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apacta,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apacta.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apacta.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apacta/apacta_test.go b/pkg/detectors/apacta/apacta_test.go
index d80e7f04b..8b6fee13b 100644
--- a/pkg/detectors/apacta/apacta_test.go
+++ b/pkg/detectors/apacta/apacta_test.go
@@ -1,120 +1,113 @@
-//go:build detectors
-// +build detectors
-
package apacta
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestApacta_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APACTA")
- inactiveSecret := testSecrets.MustGetField("APACTA_INACTIVE")
+var (
+ validPattern = "abcd1234-ef56-gh78-ij90-klmn1234opqr"
+ complexPattern = `
+ func main() {
+ url := "https://api.example.com/v1/resource"
+
+ // Create a new request with the secret as a header
+ req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte("{}")))
+ if err != nil {
+ fmt.Println("Error creating request:", err)
+ return
+ }
+
+ apactaSecret := "Bearer abcd1234-ef56-gh78-ij90-klmn1234opqr"
+ req.Header.Set("Authorization", apactaSecret)
+
+ // Perform the request
+ client := &http.Client{}
+ resp, _ := client.Do(req)
+ defer resp.Body.Close()
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ // Check response status
+ if resp.StatusCode == http.StatusOK {
+ fmt.Println("Request successful!")
+ } else {
+ fmt.Println("Request failed with status:", resp.Status)
+ }
}
+ `
+ invalidPattern = "abcD$1234-ef56-gH78-ij90-klmn1234opqr"
+)
+
+func TestApacta_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apacta secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apacta,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("apacta credentials: %s", validPattern),
+ want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apacta secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apacta,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - complex",
+ input: complexPattern,
+ want: []string{"abcd1234-ef56-gh78-ij90-klmn1234opqr"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("apacta credentials: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apacta.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apacta.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/api2cart/api2cart_integration_test.go b/pkg/detectors/api2cart/api2cart_integration_test.go
new file mode 100644
index 000000000..f0243d52b
--- /dev/null
+++ b/pkg/detectors/api2cart/api2cart_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package api2cart
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApi2Cart_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("API2CART")
+ inactiveSecret := testSecrets.MustGetField("API2CART_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a api2cart secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Api2Cart,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a api2cart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Api2Cart,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Api2Cart.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Api2Cart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/api2cart/api2cart_test.go b/pkg/detectors/api2cart/api2cart_test.go
index f0243d52b..c3fa39be9 100644
--- a/pkg/detectors/api2cart/api2cart_test.go
+++ b/pkg/detectors/api2cart/api2cart_test.go
@@ -1,120 +1,93 @@
-//go:build detectors
-// +build detectors
-
package api2cart
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "0123456789abcdef0123456789abcdef"
+ complexPattern = `
+ To integrate with API2Cart, ensure you have the following credentials in your configuration file. Your API2CART key is 1234567890abcdef1234567890abcdef, which you will need to access the API securely.
+
+ The following endpoints are available for your use:
+ - Get Products: https://api.api2cart.com/v1.0/products/get
+ - Add Product: https://api.api2cart.com/v1.0/products/add
+ `
+ invalidPattern = "0123456789Gbcde^0123456789abcdef"
)
-func TestApi2Cart_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("API2CART")
- inactiveSecret := testSecrets.MustGetField("API2CART_INACTIVE")
+func TestApi2Cart_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a api2cart secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Api2Cart,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("api2cart credentials: %s", validPattern),
+ want: []string{"0123456789abcdef0123456789abcdef"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a api2cart secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Api2Cart,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern -complex",
+ input: complexPattern,
+ want: []string{"1234567890abcdef1234567890abcdef"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("api2cart credentials: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Api2Cart.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Api2Cart.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apideck/apideck_integration_test.go b/pkg/detectors/apideck/apideck_integration_test.go
new file mode 100644
index 000000000..30eb2fa7c
--- /dev/null
+++ b/pkg/detectors/apideck/apideck_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package apideck
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApiDeck_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APIDECK_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("APIDECK_INACTIVE")
+ id := testSecrets.MustGetField("APIDECK_APPID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ApiDeck,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ApiDeck,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ApiDeck.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ApiDeck.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apideck/apideck_test.go b/pkg/detectors/apideck/apideck_test.go
index 30eb2fa7c..268ad1b0b 100644
--- a/pkg/detectors/apideck/apideck_test.go
+++ b/pkg/detectors/apideck/apideck_test.go
@@ -1,121 +1,85 @@
-//go:build detectors
-// +build detectors
-
package apideck
import (
"context"
"fmt"
+ "strings"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validKeyPattern = "sk_live_abc123def456ghi789jklmno123456pqrstuvwxyz-1234567890123456789012345678901234567890AAui09pU8as"
+ validIdPattern = "apideck_id: 1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p7q8r9s0t"
+ invalidPattern = strings.ReplaceAll(validKeyPattern, "sk_live", "sk_inactive") + strings.ReplaceAll(validIdPattern, "1", "$")
)
-func TestApiDeck_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APIDECK_TOKEN")
- inactiveSecret := testSecrets.MustGetField("APIDECK_INACTIVE")
- id := testSecrets.MustGetField("APIDECK_APPID")
+func TestApiDeck_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ApiDeck,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apideck secret %s within apideckid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
+ name: "valid pattern",
+ input: fmt.Sprintf("credentials: %s \n %s", validKeyPattern, validIdPattern),
+ want: []string{
+ validKeyPattern + "1a2b3c4d5e6f7g8h9i0j1k2l3m4n5o6p7q8r9s0t",
},
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ApiDeck,
- Verified: false,
- },
- },
- wantErr: false,
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("credentials: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ApiDeck.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ApiDeck.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apiflash/apiflash_integration_test.go b/pkg/detectors/apiflash/apiflash_integration_test.go
new file mode 100644
index 000000000..c457a4722
--- /dev/null
+++ b/pkg/detectors/apiflash/apiflash_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package apiflash
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApiflash_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APIFLASH")
+ url := testSecrets.MustGetField("API_URL")
+ inactiveSecret := testSecrets.MustGetField("APIFLASH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apiflash,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apiflash,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apiflash.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apiflash.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apiflash/apiflash_test.go b/pkg/detectors/apiflash/apiflash_test.go
index c457a4722..cae1a3b72 100644
--- a/pkg/detectors/apiflash/apiflash_test.go
+++ b/pkg/detectors/apiflash/apiflash_test.go
@@ -1,121 +1,84 @@
-//go:build detectors
-// +build detectors
-
package apiflash
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ apiflash_key: a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6
+ apiflash_url: abc123XYZ456def789ghijklm
+ `
+ invalidPattern = "0123456789Gbcde^0123456789abcdef"
)
-func TestApiflash_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APIFLASH")
- url := testSecrets.MustGetField("API_URL")
- inactiveSecret := testSecrets.MustGetField("APIFLASH_INACTIVE")
+func TestApiFlash_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apiflash,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("apiflash credentials: %s", validPattern),
+ want: []string{"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6abc123XYZ456def789ghijklm"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apiflash secret %s within apiflash %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apiflash,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("apiflash credentials: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apiflash.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apiflash.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apifonica/apifonica_integration_test.go b/pkg/detectors/apifonica/apifonica_integration_test.go
new file mode 100644
index 000000000..8df2c4cb5
--- /dev/null
+++ b/pkg/detectors/apifonica/apifonica_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package apifonica
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApifonica_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APIFONICA_SECRET")
+ token := testSecrets.MustGetField("APIFONICA_ID")
+ inactiveSecret := testSecrets.MustGetField("APIFONICA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s", secret, token)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ApiFonica,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s but not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ApiFonica,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apifonica.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apifonica.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apifonica/apifonica_test.go b/pkg/detectors/apifonica/apifonica_test.go
index 8df2c4cb5..684ce6556 100644
--- a/pkg/detectors/apifonica/apifonica_test.go
+++ b/pkg/detectors/apifonica/apifonica_test.go
@@ -1,121 +1,81 @@
-//go:build detectors
-// +build detectors
-
package apifonica
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "abc123def45-gh67-ij89-kl01-mn234op56789"
+ invalidPattern = "0123456789Gbcde^0123456789abcdef"
)
-func TestApifonica_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APIFONICA_SECRET")
- token := testSecrets.MustGetField("APIFONICA_ID")
- inactiveSecret := testSecrets.MustGetField("APIFONICA_INACTIVE")
+func TestApiFonica_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s", secret, token)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ApiFonica,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("apifonica credentials: %s", validPattern),
+ want: []string{"abc123def45-gh67-ij89-kl01-mn234op56789"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apifonica secret %s within apifonica token %s but not valid", inactiveSecret, token)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ApiFonica,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("apifonica credentials: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apifonica.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apifonica.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apify/apify_integration_test.go b/pkg/detectors/apify/apify_integration_test.go
new file mode 100644
index 000000000..79e866953
--- /dev/null
+++ b/pkg/detectors/apify/apify_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package apify
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApify_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APIFY")
+ inactiveSecret := testSecrets.MustGetField("APIFY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apify secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apify,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apify,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apify.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apify/apify_test.go b/pkg/detectors/apify/apify_test.go
index 79e866953..15798000a 100644
--- a/pkg/detectors/apify/apify_test.go
+++ b/pkg/detectors/apify/apify_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package apify
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "apify_api_ab12CD34ef56GH78ij90KL12mn34OP56qr78"
+ invalidPattern = "apify_api_ab12C#34ef56GH7$ij90KL12mn34OP56qr7)"
)
-func TestApify_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APIFY")
- inactiveSecret := testSecrets.MustGetField("APIFY_INACTIVE")
+func TestApiFy_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apify secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apify,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{validPattern},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apify secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apify,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apify.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apilayer/apilayer_integration_test.go b/pkg/detectors/apilayer/apilayer_integration_test.go
new file mode 100644
index 000000000..ef4b0fe2f
--- /dev/null
+++ b/pkg/detectors/apilayer/apilayer_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package apilayer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApilayer_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APILAYER")
+ inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apilayer,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apilayer,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apilayer/apilayer_test.go b/pkg/detectors/apilayer/apilayer_test.go
index e764f3db1..7c369b411 100644
--- a/pkg/detectors/apilayer/apilayer_test.go
+++ b/pkg/detectors/apilayer/apilayer_test.go
@@ -2,116 +2,79 @@ package apilayer
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "apilayerkey: abcdef1234567890abcdef1234567890"
+ invalidPattern = "apilayerkey: abcdef123$5678*0abcde()1234567890)"
)
-func TestApilayer_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APILAYER")
- inactiveSecret := testSecrets.MustGetField("APILAYER_INACTIVE")
+func TestApiLayer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apilayer secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apilayer,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apilayer secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apilayer,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"abcdef1234567890abcdef1234567890"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apilayer.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apilayer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apimatic/apimatic.go b/pkg/detectors/apimatic/apimatic.go
index 2d9eb5a07..1198e6707 100644
--- a/pkg/detectors/apimatic/apimatic.go
+++ b/pkg/detectors/apimatic/apimatic.go
@@ -2,17 +2,17 @@ package apimatic
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
- "strings"
"time"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@@ -23,8 +23,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic"}) + `\b([a-zA-Z0-9]{3,20}@[a-zA-Z0-9]{2,12}.[a-zA-Z0-9]{2,5})\b`)
- passPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic"}) + `\b([a-z0-9-\S]{8,32})\b`)
+ apiKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimatic", "apikey"}) + `\b([a-zA-Z0-9_-]{64})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -37,47 +36,43 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
- passMatches := passPat.FindAllStringSubmatch(dataStr, -1)
+ uniqueApiKeys := make(map[string]struct{})
+ for _, matches := range apiKeyPat.FindAllStringSubmatch(dataStr, -1) {
+ uniqueApiKeys[matches[1]] = struct{}{}
+ }
- for _, match := range matches {
- if len(match) != 2 {
- continue
+ for apiKey := range uniqueApiKeys {
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_APIMatic,
+ Raw: []byte(apiKey),
}
- userPatMatch := strings.TrimSpace(match[1])
- for _, idMatch := range passMatches {
- if len(idMatch) != 2 {
+ if verify {
+ timeout := 10 * time.Second
+ client.Timeout = timeout
+
+ // api docs: https://docs.apimatic.io/platform-api/#/http/api-endpoints/code-generation-external-apis/list-all-code-generations
+ req, err := http.NewRequestWithContext(ctx, "GET", "https://api.apimatic.io/code-generations", http.NoBody)
+ if err != nil {
continue
}
- passPatMatch := strings.TrimSpace(idMatch[1])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_APIMatic,
- Raw: []byte(userPatMatch),
- RawV2: []byte(userPatMatch + passPatMatch),
- }
- if verify {
- timeout := 10 * time.Second
- client.Timeout = timeout
- req, err := http.NewRequestWithContext(ctx, "GET", "https://www.apimatic.io/api/code-generations", nil)
- if err != nil {
- continue
- }
- req.SetBasicAuth(userPatMatch, passPatMatch)
- res, err := client.Do(req)
-
- if err == nil {
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- s1.Verified = true
- }
+ // authentication documentation: https://docs.apimatic.io/platform-api/#/http/guides/authentication
+ req.Header.Set("Authorization", "X-Auth-Key "+apiKey)
+ resp, err := client.Do(req)
+ if err == nil {
+ defer resp.Body.Close()
+ if resp.StatusCode == 200 {
+ s1.Verified = true
}
+ } else {
+ // if any error happens during api request capture that as verificaiton error
+ s1.SetVerificationError(err, apiKey)
}
-
- results = append(results, s1)
}
+
+ results = append(results, s1)
+
}
return results, nil
}
diff --git a/pkg/detectors/apimatic/apimatic_integration_test.go b/pkg/detectors/apimatic/apimatic_integration_test.go
new file mode 100644
index 000000000..7995bfc48
--- /dev/null
+++ b/pkg/detectors/apimatic/apimatic_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package apimatic
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAPIMatic_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APIMATIC")
+ pass := testSecrets.MustGetField("APIMATIC_PASS")
+ inactiveSecret := testSecrets.MustGetField("APIMATIC_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s", secret, pass)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_APIMatic,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s but not valid", inactiveSecret, pass)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_APIMatic,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("APIMatic.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("APIMatic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apimatic/apimatic_test.go b/pkg/detectors/apimatic/apimatic_test.go
index 7995bfc48..09a35794d 100644
--- a/pkg/detectors/apimatic/apimatic_test.go
+++ b/pkg/detectors/apimatic/apimatic_test.go
@@ -1,121 +1,86 @@
-//go:build detectors
-// +build detectors
-
package apimatic
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAPIMatic_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APIMATIC")
- pass := testSecrets.MustGetField("APIMATIC_PASS")
- inactiveSecret := testSecrets.MustGetField("APIMATIC_INACTIVE")
+var (
+ validPattern = `
+ func validateApiMatic() bool {
+ apiMaticKey := "rc6iLoUEFGGAWNLsuBJnmsh4tZB-oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"
- type args struct {
- ctx context.Context
- data []byte
- verify bool
+ return isActive(apiMaticKey, apiMaticPass)
}
+ `
+ invalidPattern = "apiMaticKey: rc6iL$UEFGGAWNLsuBJnmsh4tZB)oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"
+)
+
+func TestApiMatic_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s", secret, pass)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_APIMatic,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"rc6iLoUEFGGAWNLsuBJnmsh4tZB-oCxcDUmc45HIPcuiQvfUEuqo8wb9YrUd2LyB"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apimatic secret %s within apimatic pass %s but not valid", inactiveSecret, pass)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_APIMatic,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("APIMatic.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("APIMatic.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apimetrics/apimetrics.go b/pkg/detectors/apimetrics/apimetrics.go
index 7cd84d366..98c9ec37f 100644
--- a/pkg/detectors/apimetrics/apimetrics.go
+++ b/pkg/detectors/apimetrics/apimetrics.go
@@ -22,7 +22,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-bA-Z0-9\S]{32})\b`)
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apimetrics"}) + `\b([a-zA-Z0-9]{32})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
diff --git a/pkg/detectors/apimetrics/apimetrics_integration_test.go b/pkg/detectors/apimetrics/apimetrics_integration_test.go
new file mode 100644
index 000000000..a16dc4093
--- /dev/null
+++ b/pkg/detectors/apimetrics/apimetrics_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package apimetrics
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApiMetrics_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APIMETRICS")
+ inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ApiMetrics,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ApiMetrics,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apimetrics/apimetrics_test.go b/pkg/detectors/apimetrics/apimetrics_test.go
index a16dc4093..93e780ba1 100644
--- a/pkg/detectors/apimetrics/apimetrics_test.go
+++ b/pkg/detectors/apimetrics/apimetrics_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package apimetrics
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "apimetricskey: 5po8TFGawiYNCc1ct4ofWkBqzIfA6IeO"
+ invalidPattern = "apimetricskey: A1b2C3d4 E5f6G7h8I9 j0K1l2M3n4O5p6"
)
-func TestApiMetrics_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APIMETRICS")
- inactiveSecret := testSecrets.MustGetField("APIMETRICS_INACTIVE")
+func TestApiMetrics_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ApiMetrics,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"5po8TFGawiYNCc1ct4ofWkBqzIfA6IeO"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apimetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ApiMetrics,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ApiMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ApiMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apitemplate/apitemplate_integration_test.go b/pkg/detectors/apitemplate/apitemplate_integration_test.go
new file mode 100644
index 000000000..ff20dbdbd
--- /dev/null
+++ b/pkg/detectors/apitemplate/apitemplate_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package apitemplate
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAPITemplate_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APITEMPLATE")
+ inactiveSecret := testSecrets.MustGetField("APITEMPLATE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_APITemplate,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_APITemplate,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("APITemplate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("APITemplate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apitemplate/apitemplate_test.go b/pkg/detectors/apitemplate/apitemplate_test.go
index ff20dbdbd..e19fce97e 100644
--- a/pkg/detectors/apitemplate/apitemplate_test.go
+++ b/pkg/detectors/apitemplate/apitemplate_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package apitemplate
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "apitemplatekey: a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T"
+ invalidPattern = "apitemplatekey: sk_0C2c3D4e5F6@7H8i9J$k1L2m3N4o5P6q7R8s9T"
)
-func TestAPITemplate_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APITEMPLATE")
- inactiveSecret := testSecrets.MustGetField("APITEMPLATE_INACTIVE")
+func TestApiTemplate_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_APITemplate,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apitemplate secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_APITemplate,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("APITemplate.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("APITemplate.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apollo/apollo_integration_test.go b/pkg/detectors/apollo/apollo_integration_test.go
new file mode 100644
index 000000000..99ea2c755
--- /dev/null
+++ b/pkg/detectors/apollo/apollo_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package apollo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApollo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APOLLO")
+ inactiveSecret := testSecrets.MustGetField("APOLLO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apollo secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apollo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apollo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apollo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apollo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apollo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apollo/apollo_test.go b/pkg/detectors/apollo/apollo_test.go
index 99ea2c755..c66ba5417 100644
--- a/pkg/detectors/apollo/apollo_test.go
+++ b/pkg/detectors/apollo/apollo_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package apollo
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "apollo: A1b2C3d4E5f6G7h8I9j0K1"
+ invalidPattern = "apollo: A1b2C3d4-5f6G7h8I9j0K1@$"
)
-func TestApollo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APOLLO")
- inactiveSecret := testSecrets.MustGetField("APOLLO_INACTIVE")
+func TestApollo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apollo secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apollo,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"A1b2C3d4E5f6G7h8I9j0K1"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apollo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apollo,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apollo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apollo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/appcues/appcues_integration_test.go b/pkg/detectors/appcues/appcues_integration_test.go
new file mode 100644
index 000000000..c46205d00
--- /dev/null
+++ b/pkg/detectors/appcues/appcues_integration_test.go
@@ -0,0 +1,122 @@
+//go:build detectors
+// +build detectors
+
+package appcues
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppcues_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPCUES")
+ user := testSecrets.MustGetField("APPCUES_USER")
+ id := testSecrets.MustGetField("APPCUES_ID")
+ inactiveSecret := testSecrets.MustGetField("APPCUES_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s", secret, user, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Appcues,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s but not valid", inactiveSecret, user, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Appcues,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Appcues.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Appcues.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/appcues/appcues_test.go b/pkg/detectors/appcues/appcues_test.go
index c46205d00..a239edfac 100644
--- a/pkg/detectors/appcues/appcues_test.go
+++ b/pkg/detectors/appcues/appcues_test.go
@@ -1,122 +1,88 @@
-//go:build detectors
-// +build detectors
-
package appcues
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ appcues abcdef12-3456-7890-1234-56789abcdef0
+ appcues abcdef123456789012345678901234567890123
+ appcues 12345
+ `
+ invalidPattern = `
+ appcues abcdef12#3456-7890-1234-56789abcdef0
+ appcues abcdef12345678901&345678901234567890123
+ appcues 123456
+ `
)
-func TestAppcues_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPCUES")
- user := testSecrets.MustGetField("APPCUES_USER")
- id := testSecrets.MustGetField("APPCUES_ID")
- inactiveSecret := testSecrets.MustGetField("APPCUES_INACTIVE")
+func TestAppCues_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s", secret, user, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Appcues,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"abcdef12-3456-7890-1234-56789abcdef0abcdef123456789012345678901234567890123"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appcues secret %s within appcues user %s and appcues id %s but not valid", inactiveSecret, user, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Appcues,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Appcues.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Appcues.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/appfollow/appfollow_integration_test.go b/pkg/detectors/appfollow/appfollow_integration_test.go
new file mode 100644
index 000000000..fd6e46745
--- /dev/null
+++ b/pkg/detectors/appfollow/appfollow_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package appfollow
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppfollow_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPFOLLOW")
+ inactiveSecret := testSecrets.MustGetField("APPFOLLOW_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appfollow secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Appfollow,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appfollow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Appfollow,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Appfollow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Appfollow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/appfollow/appfollow_test.go b/pkg/detectors/appfollow/appfollow_test.go
index fd6e46745..dbae86cdf 100644
--- a/pkg/detectors/appfollow/appfollow_test.go
+++ b/pkg/detectors/appfollow/appfollow_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package appfollow
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.333HbjEo1oxVUFcASR0sQ8cMuIJRLcMd5H9iJWDbovCw6ESjNtuoEMQQGPN9aSoxponxrTPvn1.btADhgNetsaUBuwoyHo5ip0Jab6N6MEBnSaT6CHiO6z"
+ invalidPattern = "eyQ0eXAiOiJMM1QiLDJhbGciOiJIUXI1NiK7.abcdefgh1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcd1234efgh5678ijkl9012.qwerty12345-ASDFG67890_zxcvb_ABCDE"
)
-func TestAppfollow_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPFOLLOW")
- inactiveSecret := testSecrets.MustGetField("APPFOLLOW_INACTIVE")
+func TestAppFollow_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appfollow secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Appfollow,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("appfollow credentail: %s", validPattern),
+ want: []string{"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.333HbjEo1oxVUFcASR0sQ8cMuIJRLcMd5H9iJWDbovCw6ESjNtuoEMQQGPN9aSoxponxrTPvn1.btADhgNetsaUBuwoyHo5ip0Jab6N6MEBnSaT6CHiO6z"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appfollow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Appfollow,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("appfollow credentail: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Appfollow.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Appfollow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/appointedd/appointedd_integration_test.go b/pkg/detectors/appointedd/appointedd_integration_test.go
new file mode 100644
index 000000000..1f2a8dfe6
--- /dev/null
+++ b/pkg/detectors/appointedd/appointedd_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package appointedd
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppointedd_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPOINTEDD")
+ inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Appointedd,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Appointedd,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/appointedd/appointedd_test.go b/pkg/detectors/appointedd/appointedd_test.go
index 3323032ef..08c498c3c 100644
--- a/pkg/detectors/appointedd/appointedd_test.go
+++ b/pkg/detectors/appointedd/appointedd_test.go
@@ -4,114 +4,78 @@ import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO=XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9=Jy6yCOXmhbpEZPfY7Y"
+ invalidPattern = "Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO^XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9$Jy6yCOXmhbpEZPfY7Y"
)
-func TestAppointedd_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPOINTEDD")
- inactiveSecret := testSecrets.MustGetField("APPOINTEDD_INACTIVE")
+func TestAppFollow_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appointedd secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Appointedd,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appointedd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Appointedd,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("appointedd credentail: %s", validPattern),
+ want: []string{"Ci0a2bSpRyFcZyEXBEr9RHzf3xXllqO=XVoh+t0L0s8T2s3MFntfWhBlovqLaqEadtuJ9=Jy6yCOXmhbpEZPfY7Y"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("appointedd credentail: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Appointedd.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Appointedd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/appoptics/appoptics_integration_test.go b/pkg/detectors/appoptics/appoptics_integration_test.go
new file mode 100644
index 000000000..a50a583b0
--- /dev/null
+++ b/pkg/detectors/appoptics/appoptics_integration_test.go
@@ -0,0 +1,128 @@
+//go:build detectors
+// +build detectors
+
+package appoptics
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppoptics_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPOPTICS")
+ inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AppOptics,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AppOptics,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/appoptics/appoptics_test.go b/pkg/detectors/appoptics/appoptics_test.go
index a50a583b0..f4c9bd448 100644
--- a/pkg/detectors/appoptics/appoptics_test.go
+++ b/pkg/detectors/appoptics/appoptics_test.go
@@ -1,127 +1,80 @@
-//go:build detectors
-// +build detectors
-
package appoptics
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "IABJPR08RmvsGrebJhr1TUdo27-KtTn0mLCKkJJqj5lyba-otXPklygO9DK62o3QSPoIJ4E"
+ invalidPattern = "IABJPR08Rmvs.GrebJhr1TUdo27_KtTn0mLCKkJJqj5lyba&otXPklygO9DK62o3QSPoIJ4E"
)
-func TestAppoptics_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPOPTICS")
- inactiveSecret := testSecrets.MustGetField("APPOPTICS_INACTIVE")
+func TestAppOptics_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appoptics secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AppOptics,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("appoptics credentail: %s", validPattern),
+ want: []string{"IABJPR08RmvsGrebJhr1TUdo27-KtTn0mLCKkJJqj5lyba-otXPklygO9DK62o3QSPoIJ4E"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appoptics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AppOptics,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("appoptics credentail: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Appoptics.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Appoptics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/appsynergy/appsynergy_integration_test.go b/pkg/detectors/appsynergy/appsynergy_integration_test.go
new file mode 100644
index 000000000..363af075e
--- /dev/null
+++ b/pkg/detectors/appsynergy/appsynergy_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package appsynergy
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAppSynergy_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPSYNERGY")
+ inactiveSecret := testSecrets.MustGetField("APPSYNERGY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AppSynergy,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AppSynergy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AppSynergy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AppSynergy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/appsynergy/appsynergy_test.go b/pkg/detectors/appsynergy/appsynergy_test.go
index 363af075e..5ca3017f1 100644
--- a/pkg/detectors/appsynergy/appsynergy_test.go
+++ b/pkg/detectors/appsynergy/appsynergy_test.go
@@ -1,120 +1,81 @@
-//go:build detectors
-// +build detectors
-
package appsynergy
import (
"context"
"fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "mg1pgwlndtq7rbk8i3kum344aso8ggp02ximdhsp8nsqasd3btxf84lz9ekfdpwo"
+ invalidPattern = "mG1pgwlndtq7rbk8i3kum344a$o8gGp02ximdhsp8nsqasd3btxf84lz9ekfdpwo"
)
-func TestAppSynergy_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPSYNERGY")
- inactiveSecret := testSecrets.MustGetField("APPSYNERGY_INACTIVE")
+func TestAppSynergy_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AppSynergy,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("appsynergy credentail: %s", validPattern),
+ want: []string{"mg1pgwlndtq7rbk8i3kum344aso8ggp02ximdhsp8nsqasd3btxf84lz9ekfdpwo"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a appsynergy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AppSynergy,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("appsynergy credentail: %s", invalidPattern),
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AppSynergy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AppSynergy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/apptivo/apptivo_integration_test.go b/pkg/detectors/apptivo/apptivo_integration_test.go
new file mode 100644
index 000000000..d86c98727
--- /dev/null
+++ b/pkg/detectors/apptivo/apptivo_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package apptivo
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestApptivo_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("APPTIVO")
+ id := testSecrets.MustGetField("APPTIVO_KEY")
+ inactiveSecret := testSecrets.MustGetField("APPTIVO_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apptivo secret %s within apptivo %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apptivo,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a apptivo secret %s within but not valid apptivo %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Apptivo,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Apptivo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Apptivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/apptivo/apptivo_test.go b/pkg/detectors/apptivo/apptivo_test.go
index d86c98727..b2c74f4c4 100644
--- a/pkg/detectors/apptivo/apptivo_test.go
+++ b/pkg/detectors/apptivo/apptivo_test.go
@@ -1,121 +1,80 @@
-//go:build detectors
-// +build detectors
-
package apptivo
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "apptivokey: 1h3l8yjo3pms2h738k8f9094rkltmgy8md-4 - apptivoID: wV2WGN-KUxROVW0mlq8kur7TbKvCTdNc"
+ invalidPattern = "apptivokey: 1h3l8yjo3pms2h738k8f9094rkltmgy8md-4"
)
-func TestApptivo_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("APPTIVO")
- id := testSecrets.MustGetField("APPTIVO_KEY")
- inactiveSecret := testSecrets.MustGetField("APPTIVO_INACTIVE")
+func TestApptivo_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apptivo secret %s within apptivo %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apptivo,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"1h3l8yjo3pms2h738k8f9094rkltmgy8md-4wV2WGN-KUxROVW0mlq8kur7TbKvCTdNc"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a apptivo secret %s within but not valid apptivo %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Apptivo,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Apptivo.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Apptivo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/artifactory/artifactory_integration_test.go b/pkg/detectors/artifactory/artifactory_integration_test.go
new file mode 100644
index 000000000..deab1d778
--- /dev/null
+++ b/pkg/detectors/artifactory/artifactory_integration_test.go
@@ -0,0 +1,168 @@
+//go:build detectors
+// +build detectors
+
+package artifactory
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestArtifactory_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ARTIFACTORY_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("ARTIFACTORY_INACTIVE")
+ appURL := testSecrets.MustGetField("ARTIFACTORY_URL")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s", secret, appURL)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
+ Verified: false,
+ }
+ r.SetVerificationError(context.DeadlineExceeded)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, real secrets, verification error due to unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
+ Verified: false,
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a artifactory secret %s but not valid on endpoint %s", inactiveSecret, appURL)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/artifactory/artifactory_test.go b/pkg/detectors/artifactory/artifactory_test.go
index deab1d778..b74c91923 100644
--- a/pkg/detectors/artifactory/artifactory_test.go
+++ b/pkg/detectors/artifactory/artifactory_test.go
@@ -1,167 +1,104 @@
-//go:build detectors
-// +build detectors
-
package artifactory
import (
"context"
"fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl / RVo8ytzB65L.jfrog.io"
+ // validPattern2 is for cloud endpoints so it does not have any JFrog endpoint
+ validPattern2 = "5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl"
+ invalidPattern = "W0RSQQ0U4sVnrNgOwIJlTOqJf06T3dl^&5YcZhIKwxTdxwpZHf9c1Usu8xNtA / rtest#y$zB65L%.jfrog.io"
)
-func TestArtifactory_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ARTIFACTORY_TOKEN")
- inactiveSecret := testSecrets.MustGetField("ARTIFACTORY_INACTIVE")
- appURL := testSecrets.MustGetField("ARTIFACTORY_URL")
+func TestArtifactory_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ cloudEndpoint string
+ useCloudEndpoint bool
+ useFoundEndpoint bool
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s", secret, appURL)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
- Verified: false,
- }
- r.SetVerificationError(context.DeadlineExceeded)
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, real secrets, verification error due to unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(500, "{}")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a artifactory secret %s and domain %s but not verified", secret, appURL)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
- Verified: false,
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 500"))
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern",
+ input: fmt.Sprintf("artifactory credentials: %s", validPattern),
+ useCloudEndpoint: false,
+ useFoundEndpoint: true,
+ want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlRVo8ytzB65L.jfrog.io"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a artifactory secret %s but not valid on endpoint %s", inactiveSecret, appURL)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern - with cloud endpoints",
+ input: fmt.Sprintf("artifactory credentials: %s", validPattern2),
+ cloudEndpoint: "cloudendpoint.jfrog.io",
+ useCloudEndpoint: true,
+ useFoundEndpoint: false,
+ want: []string{"5YcZhIKwxTdxwpZHf9c1Usu8xNtAklRsqWYXWf2qmjW0RSQQ0U4sVnrNgOwIJlTOqJf06T3dlcloudendpoint.jfrog.io"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: fmt.Sprintf("artifactory credentials: %s", invalidPattern),
+ useFoundEndpoint: true,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Artifactory.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ // this detector use endpoint customizer interface so we need to enable them based on test case
+ d.UseFoundEndpoints(test.useFoundEndpoint)
+ d.UseCloudEndpoint(test.useCloudEndpoint)
+ // if test case provide cloud endpoint use it
+ if test.useCloudEndpoint && test.cloudEndpoint != "" {
+ d.SetCloudEndpoint(test.cloudEndpoint)
+ }
+
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Artifactory.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/artsy/artsy_integration_test.go b/pkg/detectors/artsy/artsy_integration_test.go
new file mode 100644
index 000000000..bf44ba496
--- /dev/null
+++ b/pkg/detectors/artsy/artsy_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package artsy
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestArtsy_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ARTSY")
+ inactiveSecret := testSecrets.MustGetField("ARTSY_INACTIVE")
+ id := testSecrets.MustGetField("ARTSY_CLIENTID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Artsy,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Artsy,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Artsy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Artsy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/artsy/artsy_test.go b/pkg/detectors/artsy/artsy_test.go
index bf44ba496..3ac7c892d 100644
--- a/pkg/detectors/artsy/artsy_test.go
+++ b/pkg/detectors/artsy/artsy_test.go
@@ -1,121 +1,80 @@
-//go:build detectors
-// +build detectors
-
package artsy
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "artsy_key: rU0K6hwGw9AeANtXrZ8FQJT9jn4sRdlj - artsy_id: hvQ2fMvUPNczDCdmzi0i"
+ invalidPattern = "artsy_key: rU:K6hwGw9AeANtXrZ8F%JT9jn4sR,lj - artsy_id: hvQ2f-vUPNczD+dmzi0i"
)
-func TestArtsy_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ARTSY")
- inactiveSecret := testSecrets.MustGetField("ARTSY_INACTIVE")
- id := testSecrets.MustGetField("ARTSY_CLIENTID")
+func TestArtsy_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Artsy,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"rU0K6hwGw9AeANtXrZ8FQJT9jn4sRdljhvQ2fMvUPNczDCdmzi0i"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a artsy secret %s within artsyid %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Artsy,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Artsy.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Artsy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/asanaoauth/asanaoauth_integration_test.go b/pkg/detectors/asanaoauth/asanaoauth_integration_test.go
new file mode 100644
index 000000000..063c55b0c
--- /dev/null
+++ b/pkg/detectors/asanaoauth/asanaoauth_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package asanaoauth
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAsanaOauth_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ASANAOAUTH_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("ASANAOAUTH_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AsanaOauth,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a asana secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AsanaOauth,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AsanaOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ got[i].AnalysisInfo = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AsanaOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/asanaoauth/asanaoauth_test.go b/pkg/detectors/asanaoauth/asanaoauth_test.go
index 063c55b0c..ff9e64ddb 100644
--- a/pkg/detectors/asanaoauth/asanaoauth_test.go
+++ b/pkg/detectors/asanaoauth/asanaoauth_test.go
@@ -1,121 +1,80 @@
-//go:build detectors
-// +build detectors
-
package asanaoauth
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "asana_key: v1e0uwkmv2ykvsu998lqe4g7ij2gwtg/xn9vh4jvsokdfaic0bn"
+ invalidPattern = "asana_key: v1e0uwkmv2ykvsu998lqe4g-ij2gwtg/xn9vh4jvsokdfaic0bn"
)
-func TestAsanaOauth_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ASANAOAUTH_TOKEN")
- inactiveSecret := testSecrets.MustGetField("ASANAOAUTH_INACTIVE")
+func TestAsanaOauth_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AsanaOauth,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"v1e0uwkmv2ykvsu998lqe4g7ij2gwtg/xn9vh4jvsokdfaic0bn"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a asana secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AsanaOauth,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AsanaOauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
- got[i].AnalysisInfo = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AsanaOauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go
new file mode 100644
index 000000000..97a61d918
--- /dev/null
+++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package asanapersonalaccesstoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ASANA_PAT")
+ inactiveSecret := testSecrets.MustGetField("ASANA_PAT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a asana secret %s within but unverified", inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AsanaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AsanaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go
index 97a61d918..8cbf64c95 100644
--- a/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go
+++ b/pkg/detectors/asanapersonalaccesstoken/asanapersonalaccesstoken_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package asanapersonalaccesstoken
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "asana_token: 594776654034514343561917591881414702593902454625364993/1724908107002616220416212965:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn"
+ invalidPattern = "asana_token: 1724908107002616220416212965%594776654034514343561917591881414702593902454625364993:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn-ij2gwtg/xn9vh4jvsokdfaic0bn"
)
-func TestAsanaPersonalAccessToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ASANA_PAT")
- inactiveSecret := testSecrets.MustGetField("ASANA_PAT_INACTIVE")
+func TestAsanaPersonalAccessToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a asana secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"594776654034514343561917591881414702593902454625364993/1724908107002616220416212965:Yv3DoiSFhtsgUwN3AcnXWjK8zabQHKSHBRHpuNKVjz3oCcpyDIdXRm3GL4SUDkTMFoTbRDCHe8tTBHxdtoXItn"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a asana secret %s within but unverified", inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AsanaPersonalAccessToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AsanaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AsanaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/assemblyai/assemblyai_integration_test.go b/pkg/detectors/assemblyai/assemblyai_integration_test.go
new file mode 100644
index 000000000..f042fb413
--- /dev/null
+++ b/pkg/detectors/assemblyai/assemblyai_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package assemblyai
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAssemblyai_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ASSEMBLYAI")
+ inactiveSecret := testSecrets.MustGetField("ASSEMBLYAI_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AssemblyAI,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AssemblyAI,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Assemblyai.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Assemblyai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/assemblyai/assemblyai_test.go b/pkg/detectors/assemblyai/assemblyai_test.go
index f042fb413..5d9dac960 100644
--- a/pkg/detectors/assemblyai/assemblyai_test.go
+++ b/pkg/detectors/assemblyai/assemblyai_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package assemblyai
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "assemblyai: mlhekyjhs96mx0r2cxbzky4jzr83fw1q"
+ invalidPattern = "assemblyai: Mlhekyjzr83fw1qr2cxbzky4jzr83f1q"
)
-func TestAssemblyai_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ASSEMBLYAI")
- inactiveSecret := testSecrets.MustGetField("ASSEMBLYAI_INACTIVE")
+func TestAssemblyAI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AssemblyAI,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"mlhekyjhs96mx0r2cxbzky4jzr83fw1q"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a assemblyai secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AssemblyAI,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Assemblyai.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Assemblyai.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/atera/atera_integration_test.go b/pkg/detectors/atera/atera_integration_test.go
new file mode 100644
index 000000000..ee90eafbc
--- /dev/null
+++ b/pkg/detectors/atera/atera_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package atera
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAtera_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("ATERA")
+ inactiveSecret := testSecrets.MustGetField("ATERA_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find an atera secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Atera,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find an atera secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Atera,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Atera.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Atera.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/atera/atera_test.go b/pkg/detectors/atera/atera_test.go
index ee90eafbc..e2000a2d6 100644
--- a/pkg/detectors/atera/atera_test.go
+++ b/pkg/detectors/atera/atera_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package atera
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "atera_key: yoo3d5pu3t4zxd6x1vhk7ykmjqarbsv1"
+ invalidPattern = "atera_key: yOO3d5pu3t4zxd6x1vhk7ykmjqarbs_1"
)
-func TestAtera_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("ATERA")
- inactiveSecret := testSecrets.MustGetField("ATERA_INACTIVE")
+func TestAtera_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find an atera secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Atera,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"yoo3d5pu3t4zxd6x1vhk7ykmjqarbsv1"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find an atera secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Atera,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Atera.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Atera.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/audd/audd.go b/pkg/detectors/audd/audd.go
index 7cfde0d94..c55cf2115 100644
--- a/pkg/detectors/audd/audd.go
+++ b/pkg/detectors/audd/audd.go
@@ -3,11 +3,12 @@ package audd
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
diff --git a/pkg/detectors/audd/audd_integration_test.go b/pkg/detectors/audd/audd_integration_test.go
new file mode 100644
index 000000000..4f1424150
--- /dev/null
+++ b/pkg/detectors/audd/audd_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package audd
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAudd_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AUDD")
+ inactiveSecret := testSecrets.MustGetField("AUDD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a audd secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Audd,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a audd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Audd,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Audd.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Audd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/audd/audd_test.go b/pkg/detectors/audd/audd_test.go
index 4f1424150..ff38207c6 100644
--- a/pkg/detectors/audd/audd_test.go
+++ b/pkg/detectors/audd/audd_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package audd
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAudd_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AUDD")
- inactiveSecret := testSecrets.MustGetField("AUDD_INACTIVE")
+var (
+ validPattern = "audd_key: 60fzzcspq2balbxn7f3hi2nvg3h07h4z"
+ invalidPattern = "audd_key: 60fuzcsoq2balxbN7f3hi2nvg3h07h4z"
+)
+
+func TestAudd_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a audd secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Audd,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"60fzzcspq2balbxn7f3hi2nvg3h07h4z"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a audd secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Audd,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Audd.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Audd.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go
new file mode 100644
index 000000000..32a0ac059
--- /dev/null
+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_integration_test.go
@@ -0,0 +1,125 @@
+//go:build detectors
+// +build detectors
+
+package auth0managementapitoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAuth0ManagementApiToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+
+ // use 2592000 for 30 days, this is the maximum allowed
+ managementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN")
+ inactiveManagementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN_INACTIVE")
+ domain := testSecrets.MustGetField("AUTH0_MANAGEMENT_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain %s", managementApiToken, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
+ Redacted: domain,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain https://%s/oauth/token within but not valid", inactiveManagementApiToken, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
+ Redacted: domain,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Auth0ManagementApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Auth0ManagementApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go
index 32a0ac059..7e098d664 100644
--- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go
+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken_test.go
@@ -1,125 +1,114 @@
-//go:build detectors
-// +build detectors
-
package auth0managementapitoken
import (
"context"
"fmt"
+ "math/rand"
+ "strings"
"testing"
"time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAuth0ManagementApiToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
+var (
+ // TODO(kashif): Refactor the fake token generation if possible
+ validPattern = generateRandomString() // this has the exact token string only which can be used in want too
+ validDomain = "QHHPu7VPj.sI.auth0.com"
+ invalidPattern = `
+ auth0_credentails:
+ apiToken: eywT2nGMZwOcbsUVBwfiRPEl8P_wnmo6XfdUoGVwxDfOSjNyqhYqFdi.KojZZOM8Ox
+ domain: QHHPu7VPj.sI.auth0.com
+ `
+)
- // use 2592000 for 30 days, this is the maximum allowed
- managementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN")
- inactiveManagementApiToken := testSecrets.MustGetField("AUTH0_MANAGEMENT_APITOKEN_INACTIVE")
- domain := testSecrets.MustGetField("AUTH0_MANAGEMENT_DOMAIN")
+func TestAuth0ManagementApitToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain %s", managementApiToken, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
- Redacted: domain,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a auth0 secret %s domain https://%s/oauth/token within but not valid", inactiveManagementApiToken, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
- Redacted: domain,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: makeFakeTokenString(validPattern, validDomain),
+ want: []string{validPattern + validDomain},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Auth0ManagementApiToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
- got[i].Raw = nil
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Auth0ManagementApiToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
- }
- })
+// makeFakeTokenString take a string token as parameter and make a string that looks like a token for testing
+func makeFakeTokenString(token, domain string) string {
+ return fmt.Sprintf("auth0:\n apiToken: %s \n domain: %s", token, domain)
+}
+
+// generateRandomString generates exactly 2001 char string for a fake token to by pass the check in detector for testing
+func generateRandomString() string {
+ const length = 2001
+ const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-"
+
+ random := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+ var builder strings.Builder
+ builder.Grow(length)
+
+ for i := 0; i < length; i++ {
+ randomChar := charset[random.Intn(len(charset))]
+ builder.WriteByte(randomChar)
}
+
+ // append ey in start as the token must start with 'ey'
+ return fmt.Sprintf("ey%s", builder.String())
}
diff --git a/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go b/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go
new file mode 100644
index 000000000..5a343487d
--- /dev/null
+++ b/pkg/detectors/auth0oauth/auth0oauth_integeration_test.go
@@ -0,0 +1,125 @@
+//go:build detectors
+// +build detectors
+
+package auth0oauth
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAuth0oauth_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+
+ domain := testSecrets.MustGetField("AUTH0_DOMAIN")
+ clientId := testSecrets.MustGetField("AUTH0_CLIENT_ID") // there is AUTH0_CLIENT_ID2 and AUTH0_CLIENT_SECRET2 pair as well
+ clientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET")
+ inactiveClientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain %s", clientId, clientSecret, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Auth0oauth,
+ Redacted: clientId,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain https://%s/oauth/token within but not valid", clientId, inactiveClientSecret, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Auth0oauth,
+ Redacted: clientId,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Auth0oauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Auth0oauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/auth0oauth/auth0oauth_test.go b/pkg/detectors/auth0oauth/auth0oauth_test.go
index 5a343487d..e573e5c03 100644
--- a/pkg/detectors/auth0oauth/auth0oauth_test.go
+++ b/pkg/detectors/auth0oauth/auth0oauth_test.go
@@ -1,125 +1,90 @@
-//go:build detectors
-// +build detectors
-
package auth0oauth
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAuth0oauth_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
+var (
+ validPattern = `
+ auth0_credentials file:
+ auth0_clientID: kYWr_tL4eYBtqIIvKfSf2-e4T9Cw1CtwE8ufoESVBB7Hi1U
+ secret: rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS-MBqIYgxH0vZaL1s5314lgPDLqHXjZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_tTx1kCXOZY_JUy
+ domain: 9-KhTIdSopSaMQ2v1YxdFEJN-HNgt7Mn7E8xkfQNqd51AzSGQu2yRaFauth0.com
+ `
+ invalidPattern = `
+ auth0_credentials file:
+ auth0_clientID: e4T9Cw1CtwE8ufoESVBB7Hi1U-e4T9Cw1CtwE8ufoESVBB7Hi1U
+ secret: MBqIYgxH0vZaL1s5314lgPDLqHX^ZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_+tTx1kCXOZY_JUy-rXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS
+ domain: 9-KhTIdSopSaMQ2v1YxdFEJN#qd51AzSGQu2yRaFauth1.com
+ `
+)
- domain := testSecrets.MustGetField("AUTH0_DOMAIN")
- clientId := testSecrets.MustGetField("AUTH0_CLIENT_ID") // there is AUTH0_CLIENT_ID2 and AUTH0_CLIENT_SECRET2 pair as well
- clientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET")
- inactiveClientSecret := testSecrets.MustGetField("AUTH0_CLIENT_SECRET_INACTIVE")
+func TestAuth0oAuth_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain %s", clientId, clientSecret, domain)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Auth0oauth,
- Redacted: clientId,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"kYWr_tL4eYBtqIIvKfSf2-e4T9Cw1CtwE8ufoESVBB7Hi1UrXwGtKCleBsaUfpchggQEAy_yhzWnqv4_GzJivBif85bqiJi3ZA63DAauoJ2PF27fvS-MBqIYgxH0vZaL1s5314lgPDLqHXjZsY59PSew63A_L6rySqcy5J3rFcGcpdeSQ_tTx1kCXOZY_JUy"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a auth0 client id %s client secret %s domain https://%s/oauth/token within but not valid", clientId, inactiveClientSecret, domain)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Auth0oauth,
- Redacted: clientId,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Auth0oauth.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Auth0oauth.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/autodesk/autodesk_integration_test.go b/pkg/detectors/autodesk/autodesk_integration_test.go
new file mode 100644
index 000000000..d3d90c83c
--- /dev/null
+++ b/pkg/detectors/autodesk/autodesk_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package autodesk
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAutodesk_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ id := testSecrets.MustGetField("AUTODESK_ID")
+ secret := testSecrets.MustGetField("AUTODESK_SECRET")
+ inactiveID := testSecrets.MustGetField("AUTODESK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Autodesk,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Autodesk,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Autodesk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Autodesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/autodesk/autodesk_test.go b/pkg/detectors/autodesk/autodesk_test.go
index d3d90c83c..cf469f817 100644
--- a/pkg/detectors/autodesk/autodesk_test.go
+++ b/pkg/detectors/autodesk/autodesk_test.go
@@ -1,121 +1,80 @@
-//go:build detectors
-// +build detectors
-
package autodesk
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "autodesk_key: 2j8Rl67MjoMruYfyIBgGzy2pxcxIQfet / autodesk_secret rHfzZhsSRruLM3Fn"
+ invalidPattern = "autodesk: 2mm8Rl67MjoMruYfyIBg5#zy2pxcxIQfet / autodesk_secret RHGklpa"
)
-func TestAutodesk_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- id := testSecrets.MustGetField("AUTODESK_ID")
- secret := testSecrets.MustGetField("AUTODESK_SECRET")
- inactiveID := testSecrets.MustGetField("AUTODESK_INACTIVE")
+func TestAutoDesk_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Autodesk,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"2j8Rl67MjoMruYfyIBgGzy2pxcxIQfetrHfzZhsSRruLM3Fn"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a autodesk secret %s within autodesk id %s but not valid", secret, inactiveID)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Autodesk,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Autodesk.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Autodesk.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/autoklose/autoklose_integration_test.go b/pkg/detectors/autoklose/autoklose_integration_test.go
new file mode 100644
index 000000000..9675d22cb
--- /dev/null
+++ b/pkg/detectors/autoklose/autoklose_integration_test.go
@@ -0,0 +1,123 @@
+//go:build detectors
+// +build detectors
+
+package autoklose
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAutoklose_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AUTOKLOSE")
+ inactiveSecret := testSecrets.MustGetField("AUTOKLOSE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a autoklose secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Autoklose,
+ Verified: true,
+ ExtraData: map[string]string{
+ "email": "[email protected]",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a autoklose secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Autoklose,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Autoklose.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Autoklose.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/autoklose/autoklose_test.go b/pkg/detectors/autoklose/autoklose_test.go
index 9675d22cb..4ed460f78 100644
--- a/pkg/detectors/autoklose/autoklose_test.go
+++ b/pkg/detectors/autoklose/autoklose_test.go
@@ -1,123 +1,80 @@
-//go:build detectors
-// +build detectors
-
package autoklose
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "autoklose credentials: KRXaU9GK3f9yHG1FS-mbwhsIXdW22epH"
+ invalidPattern = "autoklose credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH"
)
-func TestAutoklose_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AUTOKLOSE")
- inactiveSecret := testSecrets.MustGetField("AUTOKLOSE_INACTIVE")
+func TestAutoKlose_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a autoklose secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Autoklose,
- Verified: true,
- ExtraData: map[string]string{
- "email": "[email protected]",
- },
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"KRXaU9GK3f9yHG1FS-mbwhsIXdW22epH"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a autoklose secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Autoklose,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Autoklose.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Autoklose.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/autopilot/autopilot_integration_test.go b/pkg/detectors/autopilot/autopilot_integration_test.go
new file mode 100644
index 000000000..b15e26cd8
--- /dev/null
+++ b/pkg/detectors/autopilot/autopilot_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package autopilot
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAutoPilot_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AUTOPILOT")
+ inactiveSecret := testSecrets.MustGetField("AUTOPILOT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a autopilot secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AutoPilot,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a autopilot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AutoPilot,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AutoPilot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AutoPilot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/autopilot/autopilot_test.go b/pkg/detectors/autopilot/autopilot_test.go
index b15e26cd8..dd5a5f67c 100644
--- a/pkg/detectors/autopilot/autopilot_test.go
+++ b/pkg/detectors/autopilot/autopilot_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package autopilot
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "autopilot credentials: 0fd87cfb1ca6c38c5f1ae5be7b0e395e"
+ invalidPattern = "autopilot credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH"
)
-func TestAutoPilot_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AUTOPILOT")
- inactiveSecret := testSecrets.MustGetField("AUTOPILOT_INACTIVE")
+func TestAutoPilot_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a autopilot secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AutoPilot,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"0fd87cfb1ca6c38c5f1ae5be7b0e395e"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a autopilot secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AutoPilot,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AutoPilot.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AutoPilot.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go
new file mode 100644
index 000000000..80664332b
--- /dev/null
+++ b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_integration_test.go
@@ -0,0 +1,117 @@
+//go:build detectors
+// +build detectors
+
+package avazapersonalaccesstoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAvazaPersonalAccessToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN")
+ inactiveSecret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a avaza secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a avaza secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AvazaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AvazaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ s.FromData(ctx, false, data)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go
index 80664332b..7ef07b611 100644
--- a/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go
+++ b/pkg/detectors/avazapersonalaccesstoken/avazapersonalaccesstoken_test.go
@@ -1,116 +1,79 @@
-//go:build detectors
-// +build detectors
-
package avazapersonalaccesstoken
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAvazaPersonalAccessToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN")
- inactiveSecret := testSecrets.MustGetField("AVAZAPERSONALACCESSTOKEN_INACTIVE")
+var (
+ validPattern = "avaza credentials: 01818612883613176996369293-f113ceb9cf4fa63dc367ab4815b0e1edf890745f"
+ invalidPattern = "avaza credentials: 01818612883613176996369293-fzz3ceb0mf4fp63dh367xb4815b0e1edf890745f"
+)
+
+func TestAvazaPersonalAccessToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a avaza secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"01818612883613176996369293-f113ceb9cf4fa63dc367ab4815b0e1edf890745f"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a avaza secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AvazaPersonalAccessToken,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AvazaPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- got[i].Raw = nil
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AvazaPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/aviationstack/aviationstack_integration_test.go b/pkg/detectors/aviationstack/aviationstack_integration_test.go
new file mode 100644
index 000000000..b716fcd19
--- /dev/null
+++ b/pkg/detectors/aviationstack/aviationstack_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package aviationstack
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAviationStack_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AVIATIONSTACK")
+ inactiveSecret := testSecrets.MustGetField("AVIATIONSTACK_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AviationStack,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AviationStack,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AviationStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("AviationStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aviationstack/aviationstack_test.go b/pkg/detectors/aviationstack/aviationstack_test.go
index b716fcd19..c76924c9a 100644
--- a/pkg/detectors/aviationstack/aviationstack_test.go
+++ b/pkg/detectors/aviationstack/aviationstack_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package aviationstack
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "aviationstack credentials: osh0kjinsc2atoaqntoy1hdjppg54449"
+ invalidPattern = "aviationstack credentials: OSh0lMjinsc2atoaqnto[]1hdjppg5449"
)
-func TestAviationStack_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AVIATIONSTACK")
- inactiveSecret := testSecrets.MustGetField("AVIATIONSTACK_INACTIVE")
+func TestAviationStack_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AviationStack,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"osh0kjinsc2atoaqntoy1hdjppg54449"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aviationstack secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AviationStack,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AviationStack.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("AviationStack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/aws/aws_integration_test.go b/pkg/detectors/aws/aws_integration_test.go
new file mode 100644
index 000000000..92ffa5c3c
--- /dev/null
+++ b/pkg/detectors/aws/aws_integration_test.go
@@ -0,0 +1,449 @@
+//go:build detectors
+// +build detectors
+
+package aws
+
+import (
+ "context"
+ "crypto/sha256"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "github.com/stretchr/testify/assert"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX"
+
+var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
+
+// Our AWS detector interacts with AWS in an (expectedly) uncommon way that triggers some odd AWS behavior. (This odd
+// behavior doesn't affect "normal" AWS use, so it's not really "broken" - it's just something that we have to work
+// around.) The AWS detector code has a long comment explaining this in more detail, but the basic issue is that AWS STS
+// is stateful, so the behavior of these tests can vary depending on which of them you run, and in which order. This
+// particular test (TestAWS_FromChunk_InvalidValidReuseIDSequence) duplicates some logic in the "big" test table in the
+// other test in this file, but extracting it in this way as well makes it fail more consistently when it's supposed to
+// fail, which is why it's extracted.
+func TestAWS_FromChunk_InvalidValidReuseIDSequence(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AWS")
+ id := testSecrets.MustGetField("AWS_ID")
+ inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")
+
+ d := scanner{}
+
+ ignoreOpts := []cmp.Option{cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")}
+
+ got, err := d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, inactiveSecret)))
+ if assert.NoError(t, err) {
+ want := []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ },
+ },
+ }
+ if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {
+ t.Errorf("AWS.FromData() (valid ID, invalid secret) diff: (-got +want)\n%s", diff)
+ }
+ }
+
+ got, err = d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, secret)))
+ if assert.NoError(t, err) {
+ want := []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: true,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",
+ },
+ },
+ }
+ if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {
+ t.Errorf("AWS.FromData() (valid secret after invalid secret using same ID) diff: (-got +want)\n%s", diff)
+ }
+ }
+}
+
+func TestAWS_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AWS")
+ id := testSecrets.MustGetField("AWS_ID")
+ inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")
+ inactiveID := id[:len(id)-3] + "XYZ"
+ hasher := sha256.New()
+ hasher.Write([]byte(inactiveSecret))
+ hash := string(hasher.Sum(nil))
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationError bool
+ }{
+ {
+ name: "found, verified",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: true,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: scanner{verificationClient: unverifiedSecretClient},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "found two, one included for every ID found",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIAZAVB57H55F3T4XYZ",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ },
+ },
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: true,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found, because unverified secret was a hash",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: true,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",
+ },
+ },
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: inactiveID,
+ ExtraData: map[string]string{
+ "account": "619888638459",
+ "resource_type": "Access key",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified, with leading +",
+ s: scanner{
+ verificationClient: unverifiedSecretClient,
+ },
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "skipped",
+ s: scanner{
+ skipIDs: map[string]struct{}{
+ "AKIAZAVB57H55F3T4BKH": {},
+ },
+ },
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, would be verified if not for http timeout",
+ s: scanner{
+ verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),
+ },
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationError: true,
+ },
+ {
+ name: "found, unverified due to unexpected http response status",
+ s: scanner{
+ verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),
+ },
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationError: true,
+ },
+ {
+ name: "found, unverified due to invalid aws_secret with valid canary access_key_id",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: canaryAccessKeyID,
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "171436882533",
+ "is_canary": "true",
+ "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationError: false,
+ },
+ {
+ name: "found, valid canary token with no verification",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)),
+ verify: false,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: canaryAccessKeyID,
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "171436882533",
+ "is_canary": "true",
+ "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
+ },
+ },
+ },
+ wantErr: false,
+ wantVerificationError: false,
+ },
+ {
+ name: "verified secret checked directly after unverified secret with same key id",
+ s: scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: true,
+ Redacted: "AKIAZAVB57H55F3T4BKH",
+ ExtraData: map[string]string{
+ "resource_type": "Access key",
+ "account": "619888638459",
+ "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "user_id": "AIDAZAVB57H5V3Q4ACRGM",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := tt.s
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationError {
+ t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := []cmp.Option{
+ cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"),
+ cmpopts.SortSlices(func(x, y detectors.Result) bool {
+ return x.Redacted < y.Redacted
+ }),
+ }
+ if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {
+ t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go
index 92ffa5c3c..576451fed 100644
--- a/pkg/detectors/aws/aws_test.go
+++ b/pkg/detectors/aws/aws_test.go
@@ -1,448 +1,89 @@
-//go:build detectors
-// +build detectors
-
package aws
import (
"context"
- "crypto/sha256"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/stretchr/testify/assert"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-const canaryAccessKeyID = "AKIASP2TPHJSQH3FJRUX"
-
-var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
-
-// Our AWS detector interacts with AWS in an (expectedly) uncommon way that triggers some odd AWS behavior. (This odd
-// behavior doesn't affect "normal" AWS use, so it's not really "broken" - it's just something that we have to work
-// around.) The AWS detector code has a long comment explaining this in more detail, but the basic issue is that AWS STS
-// is stateful, so the behavior of these tests can vary depending on which of them you run, and in which order. This
-// particular test (TestAWS_FromChunk_InvalidValidReuseIDSequence) duplicates some logic in the "big" test table in the
-// other test in this file, but extracting it in this way as well makes it fail more consistently when it's supposed to
-// fail, which is why it's extracted.
-func TestAWS_FromChunk_InvalidValidReuseIDSequence(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AWS")
- id := testSecrets.MustGetField("AWS_ID")
- inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")
-
- d := scanner{}
-
- ignoreOpts := []cmp.Option{cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError")}
-
- got, err := d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, inactiveSecret)))
- if assert.NoError(t, err) {
- want := []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- },
- },
+var (
+ validPattern = `
+ aws credentials{
+ id: ABIAS9L8MS5IPHTZPPUQ
+ secret: .v2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63;
}
- if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {
- t.Errorf("AWS.FromData() (valid ID, invalid secret) diff: (-got +want)\n%s", diff)
+ `
+ invalidPattern = `
+ aws credentials{
+ id: AKIAs9L8MS5iPHTZPPUQ
+ secret: $YenOG.PKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63;
}
- }
-
- got, err = d.FromData(ctx, true, []byte(fmt.Sprintf("aws %s %s", id, secret)))
- if assert.NoError(t, err) {
- want := []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: true,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- "user_id": "AIDAZAVB57H5V3Q4ACRGM",
- },
- },
- }
- if diff := cmp.Diff(got, want, ignoreOpts...); diff != "" {
- t.Errorf("AWS.FromData() (valid secret after invalid secret using same ID) diff: (-got +want)\n%s", diff)
- }
- }
-}
+ `
+)
-func TestAWS_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AWS")
- id := testSecrets.MustGetField("AWS_ID")
- inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")
- inactiveID := id[:len(id)-3] + "XYZ"
- hasher := sha256.New()
- hasher.Write([]byte(inactiveSecret))
- hash := string(hasher.Sum(nil))
+func TestAWS_Pattern(t *testing.T) {
+ d := scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationError bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: true,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- "user_id": "AIDAZAVB57H5V3Q4ACRGM",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: scanner{verificationClient: unverifiedSecretClient},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- {
- name: "found two, one included for every ID found",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: "AKIAZAVB57H55F3T4XYZ",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- },
- },
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: true,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- "user_id": "AIDAZAVB57H5V3Q4ACRGM",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "not found, because unverified secret was a hash",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"ABIAS9L8MS5IPHTZPPUQv2QPKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63"},
},
{
- name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: true,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- "user_id": "AIDAZAVB57H5V3Q4ACRGM",
- },
- },
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: inactiveID,
- ExtraData: map[string]string{
- "account": "619888638459",
- "resource_type": "Access key",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified, with leading +",
- s: scanner{
- verificationClient: unverifiedSecretClient,
- },
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "skipped",
- s: scanner{
- skipIDs: map[string]struct{}{
- "AKIAZAVB57H55F3T4BKH": {},
- },
- },
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- wantErr: false,
- },
- {
- name: "found, would be verified if not for http timeout",
- s: scanner{
- verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),
- },
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- },
- },
- },
- wantErr: false,
- wantVerificationError: true,
- },
- {
- name: "found, unverified due to unexpected http response status",
- s: scanner{
- verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),
- },
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- },
- },
- },
- wantErr: false,
- wantVerificationError: true,
- },
- {
- name: "found, unverified due to invalid aws_secret with valid canary access_key_id",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", inactiveSecret, canaryAccessKeyID)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: canaryAccessKeyID,
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "171436882533",
- "is_canary": "true",
- "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
- },
- },
- },
- wantErr: false,
- wantVerificationError: false,
- },
- {
- name: "found, valid canary token with no verification",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, canaryAccessKeyID)),
- verify: false,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: canaryAccessKeyID,
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "171436882533",
- "is_canary": "true",
- "message": "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries",
- },
- },
- },
- wantErr: false,
- wantVerificationError: false,
- },
- {
- name: "verified secret checked directly after unverified secret with same key id",
- s: scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AWS,
- Verified: true,
- Redacted: "AKIAZAVB57H55F3T4BKH",
- ExtraData: map[string]string{
- "resource_type": "Access key",
- "account": "619888638459",
- "arn": "arn:aws:iam::619888638459:user/trufflehog-aws-detector-tester",
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- "user_id": "AIDAZAVB57H5V3Q4ACRGM",
- },
- },
- },
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := tt.s
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if (got[i].VerificationError() != nil) != tt.wantVerificationError {
- t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError())
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
+ return
}
- ignoreOpts := []cmp.Option{
- cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "verificationError"),
- cmpopts.SortSlices(func(x, y detectors.Result) bool {
- return x.Redacted < y.Redacted
- }),
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
}
- if diff := cmp.Diff(got, tt.want, ignoreOpts...); diff != "" {
- t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/awssessionkeys/awssessionkeys_test.go b/pkg/detectors/awssessionkeys/awssessionkeys_test.go
new file mode 100644
index 000000000..8ed4028e6
--- /dev/null
+++ b/pkg/detectors/awssessionkeys/awssessionkeys_test.go
@@ -0,0 +1,91 @@
+package awssessionkey
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ aws credentials{
+ id: ASIABBKK02W42Q3IPSPG
+ secret: fkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8O
+ session: >aSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa~
+ }
+ `
+ invalidPattern = `
+ aws credentials{
+ id: ASIABBKK02W42Q3IPSPG
+ secret: $YenOG.PKHl7LcdVYsjaR4LgQiZ1zw3MAnMyiondXC63;
+ }
+ `
+)
+
+func TestAWSSessionKey_Pattern(t *testing.T) {
+ d := scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+
+ tests := []struct {
+ name string
+ input string
+ want []string
+ }{
+ {
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"ASIABBKK02W42Q3IPSPGfkhIiUwQY32Zu9e4a86g9r3WpTzfE1aXljVcgn8OaSqfp/GTZbJP+=tXPNCZ9GoveoM0vgxtlYXdzPQ2uYNMPPgUkt0VT7SoTLasAo7iVqWWREOUC6DEenlcgDEKyzIEgQ=W5Ju/b9K/Z176uD2HJYCfq/lyowHtt5PvJi7LR=uf/urSorGbTcqNUvP=i42YP1Ps/4F6He9hQA1io3EAGBC3ICGHXWf2IlvFoTNUyPTqhjnPEKMWZ42jblqNA=dD7hLpzNXmmGhdLCjy99XK8+gjHdZHkOeD/FIjRPRZ7Jl0tdwdqFEwzRVCzL2uelMVMd3UaZ+d4I4Kf+J464piO//jxx48Fs/mG3zr5ba9m2S+6gvUZJq4j+0uJ+jf6cG/x2G9XSybqYQRwvxfNquKB4TcKiGVH5+ZbJT4ASkARadwoSPMGfvMPje+X2zAziSzXfsxYfIQKf6iJ9p7VavlDGi+Acr4kwFXW5IfQs4uGk6AVQFsoZK3o1hhLOkuOwWQEWhDQGNLXwJbFqXfELOnUQvM0Z5NUm46bjAAi4g+X9gLPNR/KjzXuuTTaWYrQEjXLb7PxS0sIttAb1w+sTXXtc1kDIsABC6KcsyGlEwji5sLkbkUa"},
+ },
+ {
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/axonaut/axonaut_integration_test.go b/pkg/detectors/axonaut/axonaut_integration_test.go
new file mode 100644
index 000000000..0d64cbc64
--- /dev/null
+++ b/pkg/detectors/axonaut/axonaut_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package axonaut
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAxonaut_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AXONAUT")
+ inactiveSecret := testSecrets.MustGetField("AXONAUT_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a axonaut secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Axonaut,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a axonaut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Axonaut,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Axonaut.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Axonaut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/axonaut/axonaut_test.go b/pkg/detectors/axonaut/axonaut_test.go
index 0d64cbc64..fdf924afc 100644
--- a/pkg/detectors/axonaut/axonaut_test.go
+++ b/pkg/detectors/axonaut/axonaut_test.go
@@ -1,120 +1,88 @@
-//go:build detectors
-// +build detectors
-
package axonaut
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ axonaut credentials{
+ key: 4ve4aj6v38uiadaq9hcgpupp2b3lh2k8
+ }
+ `
+ invalidPattern = `
+ axonaut credentials{
+ key: ASIABBKK02W42Q3IPSPG
+ }
+ `
)
-func TestAxonaut_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AXONAUT")
- inactiveSecret := testSecrets.MustGetField("AXONAUT_INACTIVE")
+func TestAxonaut_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a axonaut secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Axonaut,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"4ve4aj6v38uiadaq9hcgpupp2b3lh2k8"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a axonaut secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Axonaut,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Axonaut.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Axonaut.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/aylien/aylien_integration_test.go b/pkg/detectors/aylien/aylien_integration_test.go
new file mode 100644
index 000000000..e1315f5f9
--- /dev/null
+++ b/pkg/detectors/aylien/aylien_integration_test.go
@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package aylien
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAylien_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AYLIEN")
+ id := testSecrets.MustGetField("AYLIEN_ID")
+ inactiveSecret := testSecrets.MustGetField("AYLIEN_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aylien,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Aylien,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Aylien.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Aylien.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/aylien/aylien_test.go b/pkg/detectors/aylien/aylien_test.go
index e1315f5f9..5f01ae03b 100644
--- a/pkg/detectors/aylien/aylien_test.go
+++ b/pkg/detectors/aylien/aylien_test.go
@@ -1,121 +1,88 @@
-//go:build detectors
-// +build detectors
-
package aylien
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ aylien credentials:
+ aylien key: cr479du2l9pkmhar8gw5hufofvwp86q9
+ aylien id: y3ejw028
+ `
+ invalidPattern = `
+ aylien credentials:
+ aylien key: cr4U9du2l9pkmhar8gw5hufofvWp86q9
+ aylien id: y3ejwA8
+ `
)
-func TestAylien_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AYLIEN")
- id := testSecrets.MustGetField("AYLIEN_ID")
- inactiveSecret := testSecrets.MustGetField("AYLIEN_INACTIVE")
+func TestAylien_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s", secret, id)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aylien,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"cr479du2l9pkmhar8gw5hufofvwp86q9y3ejw028"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a aylien secret %s within aylien %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Aylien,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Aylien.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Aylien.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/ayrshare/ayrshare_integration_test.go b/pkg/detectors/ayrshare/ayrshare_integration_test.go
new file mode 100644
index 000000000..da11fac95
--- /dev/null
+++ b/pkg/detectors/ayrshare/ayrshare_integration_test.go
@@ -0,0 +1,120 @@
+//go:build detectors
+// +build detectors
+
+package ayrshare
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAyrshare_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AYRSHARE_TOKEN")
+ inactiveSecret := testSecrets.MustGetField("AYRSHARE_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ayrshare,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Ayrshare,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Ayrshare.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Ayrshare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/ayrshare/ayrshare_test.go b/pkg/detectors/ayrshare/ayrshare_test.go
index da11fac95..9eb55166f 100644
--- a/pkg/detectors/ayrshare/ayrshare_test.go
+++ b/pkg/detectors/ayrshare/ayrshare_test.go
@@ -1,120 +1,80 @@
-//go:build detectors
-// +build detectors
-
package ayrshare
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = "ayrshare credentials: 2FTJTA1C-BXO0DV4J-HGTP9E62-QHQSILY1"
+ invalidPattern = "ayrshare credentials: KRXaU9GK3f[yHG1FS$]bwhsIXdW22epH"
)
-func TestAyrshare_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AYRSHARE_TOKEN")
- inactiveSecret := testSecrets.MustGetField("AYRSHARE_INACTIVE")
+func TestAyrShare_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ayrshare,
- Verified: true,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"2FTJTA1C-BXO0DV4J-HGTP9E62-QHQSILY1"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a ayrshare secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Ayrshare,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Ayrshare.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Ayrshare.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/azure/azure_integration_test.go b/pkg/detectors/azure/azure_integration_test.go
new file mode 100644
index 000000000..9ecc18ab1
--- /dev/null
+++ b/pkg/detectors/azure/azure_integration_test.go
@@ -0,0 +1,113 @@
+//go:build detectors
+// +build detectors
+
+package azure
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzure_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_SECRET")
+ secretInactive := testSecrets.MustGetField("AZURE_INACTIVE")
+ id := testSecrets.MustGetField("AZURE_ID")
+ tenantId := testSecrets.MustGetField("AZURE_TENANT_ID")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`
+ tenant_id=%s
+ client_id=%s
+ client_secret=%s
+ client_secret=%s
+ `, tenantId, id, secretInactive, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Azure,
+ Redacted: id,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`
+ tenant_id=%s
+ client_id=%s
+ client_secret=%s
+ `, tenantId, id, secretInactive)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_Azure,
+ Redacted: id,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azure/azure_test.go b/pkg/detectors/azure/azure_test.go
index 9ecc18ab1..74b2fadb5 100644
--- a/pkg/detectors/azure/azure_test.go
+++ b/pkg/detectors/azure/azure_test.go
@@ -1,112 +1,89 @@
-//go:build detectors
-// +build detectors
-
package azure
import (
"context"
- "fmt"
"testing"
- "time"
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/google/go-cmp/cmp"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAzure_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AZURE_SECRET")
- secretInactive := testSecrets.MustGetField("AZURE_INACTIVE")
- id := testSecrets.MustGetField("AZURE_ID")
- tenantId := testSecrets.MustGetField("AZURE_TENANT_ID")
+var (
+ validPattern = `
+ azure credentials:
+ azureClientID: clientid9304d5df4-aac1-6117-552c-7f70c89a40d9
+ azureTenant: tenant_idid9304d5df4-aac1-6117-552c-7f70c89a40d9
+ azureClientSecret: clientsecretY_0w|[cGpan41k6ng.ol414sp4ccw2v_rkfmbs537i
+ `
+ invalidPattern = `
+ azure credentials:
+ azureClientID: 9304d5df4-aac1-6117-552c-7f70c89a
+ azureTenant: id9304d5df4-aac1-6117-55-7f70c89a40d9
+ azureClientSecret: Y_0w|[cGpan41k6ng.
+ `
+)
+
+func TestAzure_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`
- tenant_id=%s
- client_id=%s
- client_secret=%s
- client_secret=%s
- `, tenantId, id, secretInactive, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Azure,
- Redacted: id,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf(`
- tenant_id=%s
- client_id=%s
- client_secret=%s
- `, tenantId, id, secretInactive)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Azure,
- Redacted: id,
- Verified: false,
- },
- },
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"304d5df4-aac1-6117-552c-7f70c89a40d9cGpan41k6ng.ol414sp4ccw2v_rkfmbs53304d5df4-aac1-6117-552c-7f70c89a40d9"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Azure.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- got[i].Raw = nil
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Azure.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/azurebatch/azurebatch_integration_test.go b/pkg/detectors/azurebatch/azurebatch_integration_test.go
new file mode 100644
index 000000000..dc92a401b
--- /dev/null
+++ b/pkg/detectors/azurebatch/azurebatch_integration_test.go
@@ -0,0 +1,130 @@
+//go:build detectors
+// +build detectors
+
+package azurebatch
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzurebatch_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ url := testSecrets.MustGetField("AZUREBATCH_URL")
+ secret := testSecrets.MustGetField("AZUREBATCH_KEY")
+ inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureBatch,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureBatch,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")
+
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azurebatch/azurebatch_test.go b/pkg/detectors/azurebatch/azurebatch_test.go
index dc92a401b..82a5ac19b 100644
--- a/pkg/detectors/azurebatch/azurebatch_test.go
+++ b/pkg/detectors/azurebatch/azurebatch_test.go
@@ -1,130 +1,88 @@
-//go:build detectors
-// +build detectors
-
package azurebatch
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ azure batch:
+ url: https://JrxlYxT+0hW.YSA.batch.azure.com
+ secret: BXIMbhBlC3=5hIbqCEKvq7opaV2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/LB0
+ `
+ invalidPattern = `
+ azure batch:
+ url: http://invalid.this.batch.azure.com
+ secret: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/
+ `
)
-func TestAzurebatch_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- url := testSecrets.MustGetField("AZUREBATCH_URL")
- secret := testSecrets.MustGetField("AZUREBATCH_KEY")
- inactiveSecret := testSecrets.MustGetField("AZUREBATCH_KEY_INACTIVE")
+func TestAzureBatch_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within", url, secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureBatch,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurebatch secret %s and %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureBatch,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"https://JrxlYxT+0hW.YSA.batch.azure.comBXIMbhBlC3=5hIbqCEKvq7opaV2ZfO0XWbcnasZmPm/AJfQqdcnt/AVmKkJ8Qw80Zc1rQDaw+2Ytxc1hDq1m/LB0"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AzureBatch.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
- }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AzureBatch.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
})
}
}
diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go
new file mode 100644
index 000000000..8ed032f97
--- /dev/null
+++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_integration_test.go
@@ -0,0 +1,129 @@
+//go:build detectors
+// +build detectors
+
+package azurecontainerregistry
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzureContainerRegistry_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ azureHost := testSecrets.MustGetField("AZURE_CR_HOST")
+ password := testSecrets.MustGetField("AZURE_CR_PASSWORD")
+ passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go
index 8ed032f97..b477796ab 100644
--- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go
+++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry_test.go
@@ -1,128 +1,87 @@
-//go:build detectors
-// +build detectors
-
package azurecontainerregistry
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ azure:
+ url: 02r0TG8OBkyxWIhSrcCMUQCazXEpVwdrTSVKCstwytZjAa4b2fKJFCy9n.azurecr.io
+ secret: np5kXaH6Qu9zVE5nudGcsm+3/qSOWt/hQ/4bP0kvk6aw98pJEjE3
+ `
+ invalidPattern = `
+ azure:
+ url: http://invalid.azurecr.io.azure.com
+ secret: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/
+ `
)
-func TestAzureContainerRegistry_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- azureHost := testSecrets.MustGetField("AZURE_CR_HOST")
- password := testSecrets.MustGetField("AZURE_CR_PASSWORD")
- passwordInactive := testSecrets.MustGetField("AZURE_CR_PASSWORD_INACTIVE")
+func TestAzureContainerRegistry_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within", azureHost, password)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
- Verified: true,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"02r0TG8OBkyxWIhSrcCMUQCazXEpVwdrTSVKCstwytZjAa4b2fKJFCy9n.azurecr.ionp5kXaH6Qu9zVE5nudGcsm+3/qSOWt/hQ/4bP0kvk6aw98pJEjE3"},
},
{
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurecontainerregistry secret %s and %s within but not valid", azureHost, passwordInactive)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureContainerRegistry,
- Verified: false,
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AzureContainerRegistry.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "Redacted", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AzureContainerRegistry.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go
new file mode 100644
index 000000000..2b7c68f3f
--- /dev/null
+++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_integration_test.go
@@ -0,0 +1,170 @@
+//go:build detectors
+// +build detectors
+
+package azuredevopspersonalaccesstoken
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzureDevopsPersonalAccessToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_DEVOPS_PAT")
+ inactiveSecret := testSecrets.MustGetField("AZURE_DEVOPS_PAT_INACTIVE")
+ org := testSecrets.MustGetField("AZURE_DEVOPS_PAT_ORG")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
+ Verified: true,
+ RawV2: []byte(secret + org),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + org),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
+ Verified: false,
+ RawV2: []byte(secret + org),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
+ Verified: false,
+ RawV2: []byte(secret + org),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AzureDevopsPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if len(got[i].RawV2) == 0 {
+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AzureDevopsPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go
index 2b7c68f3f..b19ef9de3 100644
--- a/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go
+++ b/pkg/detectors/azuredevopspersonalaccesstoken/azuredevopspersonalaccesstoken_test.go
@@ -1,169 +1,87 @@
-//go:build detectors
-// +build detectors
-
package azuredevopspersonalaccesstoken
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ azure:
+ azure_key: uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un
+ azure_org_id: WOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG
+ `
+ invalidPattern = `
+ azure:
+ azure_key: uie5tff7m5H5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8un
+ azure_org_id: LOKi
+ `
)
-func TestAzureDevopsPersonalAccessToken_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AZURE_DEVOPS_PAT")
- inactiveSecret := testSecrets.MustGetField("AZURE_DEVOPS_PAT_INACTIVE")
- org := testSecrets.MustGetField("AZURE_DEVOPS_PAT_ORG")
+func TestAzureDevopsPersonalAccessToken_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
- Verified: true,
- RawV2: []byte(secret + org),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within but not valid", inactiveSecret, org)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
- Verified: false,
- RawV2: []byte(inactiveSecret + org),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"uie5tff7m5h5lqnqjhaltetqli90a08p6dhv9rn59uo30jgzw8unWOkQXnjSxCyioEJRa8R6J39cN4Xfyy8CWl1BZksHYsevxVBFzG"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
- Verified: false,
- RawV2: []byte(secret + org),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure organization %s within", secret, org)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureDevopsPersonalAccessToken,
- Verified: false,
- RawV2: []byte(secret + org),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AzureDevopsPersonalAccessToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if len(got[i].RawV2) == 0 {
- t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AzureDevopsPersonalAccessToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go b/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go
new file mode 100644
index 000000000..2d9881ed7
--- /dev/null
+++ b/pkg/detectors/azurefunctionkey/azurefunctionkey_integration_test.go
@@ -0,0 +1,170 @@
+//go:build detectors
+// +build detectors
+
+package azurefunctionkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzureFunctionKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_FUNCTION_KEY")
+ inactiveSecret := testSecrets.MustGetField("AZURE_FUNCTION_KEY_INACTIVE")
+ url := testSecrets.MustGetField("AZURE_FUNCTION_URL")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,
+ Verified: true,
+ RawV2: []byte(secret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,
+ Verified: false,
+ RawV2: []byte(secret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureFunctionKey,
+ Verified: false,
+ RawV2: []byte(secret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AzureFunctionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if len(got[i].RawV2) == 0 {
+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AzureFunctionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go b/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go
index 2d9881ed7..f284a28d7 100644
--- a/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go
+++ b/pkg/detectors/azurefunctionkey/azurefunctionkey_test.go
@@ -1,169 +1,90 @@
-//go:build detectors
-// +build detectors
-
package azurefunctionkey
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ azure:
+ azureURL: https://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E
+ azureFunctionkey: B8sm0KyfL1y8vPH3IDTdefevHBCGK33-=
+ `
+ invalidPattern = `
+ azure:
+ azureURL: http://invalid.azurecr.io.azure.com
+ azureFunctionkey: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/
+ `
)
-func TestAzureFunctionKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AZURE_FUNCTION_KEY")
- inactiveSecret := testSecrets.MustGetField("AZURE_FUNCTION_KEY_INACTIVE")
- url := testSecrets.MustGetField("AZURE_FUNCTION_URL")
+func TestAzureFunctionKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureFunctionKey,
- Verified: true,
- RawV2: []byte(secret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation
- verify: true,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{
+ "azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0Ehttps://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E",
+ "B8sm0KyfL1y8vPH3IDTdefevHBCGK33https://z1dUSi5T.azurewebsites.net/api/W8anB5J4uQi-v3Dcd6p7ySE0E",
},
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureFunctionKey,
- Verified: false,
- RawV2: []byte(inactiveSecret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureFunctionKey,
- Verified: false,
- RawV2: []byte(secret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s azure url %s", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureFunctionKey,
- Verified: false,
- RawV2: []byte(secret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AzureFunctionKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if len(got[i].RawV2) == 0 {
- t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AzureFunctionKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go
new file mode 100644
index 000000000..c59e16cee
--- /dev/null
+++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_integration_test.go
@@ -0,0 +1,170 @@
+//go:build detectors
+// +build detectors
+
+package azuresearchadminkey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzureSearchAdminKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY")
+ inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_INACTIVE")
+ service := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_SERVICE")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
+ Verified: true,
+ RawV2: []byte(secret + service),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within but not valid", inactiveSecret, service)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + service),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
+ Verified: false,
+ RawV2: []byte(secret + service),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
+ Verified: false,
+ RawV2: []byte(secret + service),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AzureSearchAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if len(got[i].RawV2) == 0 {
+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AzureSearchAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go
index c59e16cee..aeb5b2094 100644
--- a/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go
+++ b/pkg/detectors/azuresearchadminkey/azuresearchadminkey_test.go
@@ -1,169 +1,87 @@
-//go:build detectors
-// +build detectors
-
package azuresearchadminkey
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+var (
+ validPattern = `
+ azure:
+ azureKey: wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJyma
+ azureService: TestingService01
+ `
+ invalidPattern = `
+ azure:
+ Key: wRRPyhjv8m6JGRujUUr-PK#a8d3rJ0mrGAxhmqf3A68OgZmlWUJyma
+ Service: TS01
+ `
)
-func TestAzureSearchAdminKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY")
- inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_INACTIVE")
- service := testSecrets.MustGetField("AZURE_SEARCH_ADMIN_KEY_SERVICE")
+func TestAzureSearchAdminKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
- Verified: true,
- RawV2: []byte(secret + service),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within but not valid", inactiveSecret, service)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
- Verified: false,
- RawV2: []byte(inactiveSecret + service),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaTestingService01", "wRRPyhjv8m6JGRujUUrPKa8d3rJ0mrGAxhmqf3A68OgZmlWUJymaazureKey"},
},
{
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
- Verified: false,
- RawV2: []byte(secret + service),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure service %s within", secret, service)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchAdminKey,
- Verified: false,
- RawV2: []byte(secret + service),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AzureSearchAdminKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if len(got[i].RawV2) == 0 {
- t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AzureSearchAdminKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go
new file mode 100644
index 000000000..1967a9b97
--- /dev/null
+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_integration_test.go
@@ -0,0 +1,170 @@
+//go:build detectors
+// +build detectors
+
+package azuresearchquerykey
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzureSearchQueryKey_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY")
+ inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_INACTIVE")
+ url := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_URL")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ wantVerificationErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
+ Verified: true,
+ RawV2: []byte(secret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
+ Verified: false,
+ RawV2: []byte(inactiveSecret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ wantVerificationErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
+ Verified: false,
+ RawV2: []byte(secret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
+ Verified: false,
+ RawV2: []byte(secret + url),
+ },
+ },
+ wantErr: false,
+ wantVerificationErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("AzureSearchQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ if len(got[i].RawV2) == 0 {
+ t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+ }
+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("AzureSearchQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go
index 1967a9b97..3af2254ec 100644
--- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go
+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey_test.go
@@ -1,169 +1,88 @@
-//go:build detectors
-// +build detectors
-
package azuresearchquerykey
import (
"context"
- "fmt"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+var (
+ validPattern = `
+ azure:
+ azure_url: https://tzyexx2ktdfhha8w1cktqzbrgv37ywtu.search.windows.net/indexes/n81wg81jogjfq93cyxfi67vy2g7vwlcqfgi
+ azure_key: OKalbM5EBt5hloqU46phTUCZqvNAlZ4S2Jd2gFUCOQ3HG0vQ2uEp
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ `
+ invalidPattern = `
+ azure:
+ url: http://invalid.azurecr.io.azure.com
+ azure_key: BXIMbhBlC3=5hIbqCEKvq7op!V2ZfO0XWbcnasZmPm/AJfQqdcnt/+2Ytxc1hDq1m/
+ `
)
-func TestAzureSearchQueryKey_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY")
- inactiveSecret := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_INACTIVE")
- url := testSecrets.MustGetField("AZURE_SEARCH_QUERY_KEY_URL")
+func TestAzureSearchQueryKey_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- wantVerificationErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
- Verified: true,
- RawV2: []byte(secret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within but not valid", inactiveSecret, url)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
- Verified: false,
- RawV2: []byte(inactiveSecret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- wantVerificationErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"OKalbM5EBt5hloqU46phTUCZqvNAlZ4S2Jd2gFUCOQ3HG0vQ2uEphttps://tzyexx2ktdfhha8w1cktqzbrgv37ywtu.search.windows.net/indexes/n81wg81jogjfq93cyxfi67vy2g7vwlcqfgi"},
},
{
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
- Verified: false,
- RawV2: []byte(secret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azure secret %s and azure url %s within", secret, url)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,
- Verified: false,
- RawV2: []byte(secret + url),
- },
- },
- wantErr: false,
- wantVerificationErr: true,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("AzureSearchQueryKey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- if len(got[i].RawV2) == 0 {
- t.Fatalf("no rawV2 secret present: \n %+v", got[i])
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if (got[i].VerificationError() != nil) != tt.wantVerificationErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("AzureSearchQueryKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}
diff --git a/pkg/detectors/azurestorage/azurestorage_integration_test.go b/pkg/detectors/azurestorage/azurestorage_integration_test.go
new file mode 100644
index 000000000..1e45f122e
--- /dev/null
+++ b/pkg/detectors/azurestorage/azurestorage_integration_test.go
@@ -0,0 +1,205 @@
+//go:build detectors
+// +build detectors
+
+package azurestorage
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestAzurestorage_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ secret := testSecrets.MustGetField("AZURE_STORAGE")
+ inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE")
+
+ accountNamePat := regexp.MustCompile(`AccountName=(?P<account_name>[^;]+);AccountKey`)
+ accountName := accountNamePat.FindStringSubmatch(secret)[1]
+ validKeyInvalidAccountName := strings.Replace(secret, accountName, "invalid", 1)
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureStorage,
+ Verified: true,
+ ExtraData: map[string]string{
+ "account_name": "teststoragebytruffle",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureStorage,
+ Verified: false,
+ ExtraData: map[string]string{
+ "account_name": "teststoragebytruffle",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ {
+ name: "found, would be verified if not for timeout",
+ s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AzureStorage,
+ Verified: false,
+ ExtraData: map[string]string{
+ "account_name": "teststoragebytruffle",
+ },
+ }
+ r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found, verified but unexpected api surface",
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
+ verify: true,
+ },
+ want: func() []detectors.Result {
+ r := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AzureStorage,
+ Verified: false,
+ ExtraData: map[string]string{
+ "account_name": "teststoragebytruffle",
+ },
+ }
+ r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret)
+ return []detectors.Result{r}
+ }(),
+ wantErr: false,
+ },
+ {
+ name: "found secret with invalid account name",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", validKeyInvalidAccountName)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AzureStorage,
+ Verified: false,
+ ExtraData: map[string]string{
+ "account_name": "invalid",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ gotErr := ""
+ if got[i].VerificationError() != nil {
+ gotErr = got[i].VerificationError().Error()
+ }
+ wantErr := ""
+ if tt.want[i].VerificationError() != nil {
+ wantErr = tt.want[i].VerificationError().Error()
+ }
+ if gotErr != wantErr {
+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ }
+ }
+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+ t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azurestorage/azurestorage_test.go b/pkg/detectors/azurestorage/azurestorage_test.go
index 343f65390..676900289 100644
--- a/pkg/detectors/azurestorage/azurestorage_test.go
+++ b/pkg/detectors/azurestorage/azurestorage_test.go
@@ -1,203 +1,85 @@
-//go:build detectors
-// +build detectors
-
package azurestorage
import (
"context"
- "fmt"
- regexp "github.com/wasilibs/go-re2"
- "strings"
"testing"
- "time"
"github.com/google/go-cmp/cmp"
- "github.com/google/go-cmp/cmp/cmpopts"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-func TestAzurestorage_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("AZURE_STORAGE")
- inactiveSecret := testSecrets.MustGetField("AZURE_STORAGE_INACTIVE")
+var (
+ validPattern = `
+ azure storage:
+ DefaultEndpointsProtocol=https;AccountName=myStorageAccount;AccountKey=myAccountKey12345==;EndpointSuffix=core.windows.net
+ `
+ invalidPattern = `
+ azure storage:
+ DefaultEndpointsProtocol=https;AccountName=;AccountKey=myAccountKey12345==;EndpointSuffix=core.ubuntu.net
+ `
+)
- accountNamePat := regexp.MustCompile(`AccountName=(?P<account_name>[^;]+);AccountKey`)
- accountName := accountNamePat.FindStringSubmatch(secret)[1]
- validKeyInvalidAccountName := strings.Replace(secret, accountName, "invalid", 1)
+func TestAzureStorage_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
+ name string
+ input string
+ want []string
}{
{
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureStorage,
- Verified: true,
- ExtraData: map[string]string{
- "account_name": "teststoragebytruffle",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureStorage,
- Verified: false,
- ExtraData: map[string]string{
- "account_name": "teststoragebytruffle",
- },
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- {
- name: "found, would be verified if not for timeout",
- s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_AzureStorage,
- Verified: false,
- ExtraData: map[string]string{
- "account_name": "teststoragebytruffle",
- },
- }
- r.SetVerificationError(fmt.Errorf("context deadline exceeded"), secret)
- return []detectors.Result{r}
- }(),
- wantErr: false,
- },
- {
- name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", secret)),
- verify: true,
- },
- want: func() []detectors.Result {
- r := detectors.Result{
- DetectorType: detectorspb.DetectorType_AzureStorage,
- Verified: false,
- ExtraData: map[string]string{
- "account_name": "teststoragebytruffle",
- },
- }
- r.SetVerificationError(fmt.Errorf("unexpected HTTP response status 404"), secret)
- return []detectors.Result{r}
- }(),
- wantErr: false,
+ name: "valid pattern",
+ input: validPattern,
+ want: []string{"myAccountKey12345=="},
},
{
- name: "found secret with invalid account name",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a azurestorage secret %s within", validKeyInvalidAccountName)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_AzureStorage,
- Verified: false,
- ExtraData: map[string]string{
- "account_name": "invalid",
- },
- },
- },
- wantErr: false,
+ name: "invalid pattern",
+ input: invalidPattern,
+ want: nil,
},
}
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Azuretorage.FromData() error = %v, wantErr %v", err, tt.wantErr)
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
return
}
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- gotErr := ""
- if got[i].VerificationError() != nil {
- gotErr = got[i].VerificationError().Error()
- }
- wantErr := ""
- if tt.want[i].VerificationError() != nil {
- wantErr = tt.want[i].VerificationError().Error()
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
}
- if gotErr != wantErr {
- t.Fatalf("wantVerificationError = %v, verification error = %v", tt.want[i].VerificationError(), got[i].VerificationError())
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
}
}
- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
- if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
- t.Errorf("Azurestorage.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
}
- })
- }
-}
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- b.ResetTimer()
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
}
})
}