Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[INS-197] Add redhatpyxis api key detector (#4995)
Muneeb Ullah Khan885fa2d6b0e8
diff --git a/main.go b/main.goindex 448b8e4a2..a9e170b67 100644--- a/main.go+++ b/main.go@@ -544,6 +544,7 @@ func run(state overseer.State, logSync func() error) {feature.UserDetectorEnabled.Store(true)feature.BraintrustDetectorEnabled.Store(true)feature.PgAnalyzeReadKeyDetectorEnabled.Store(true)+ feature.RedHatPyxisDetectorEnabled.Store(true)conf := &config.Config{}if *configFilename != "" {diff --git a/pkg/detectors/redhatpyxis/redhatpyxis.go b/pkg/detectors/redhatpyxis/redhatpyxis.gonew file mode 100644index 000000000..46e8e869f--- /dev/null+++ b/pkg/detectors/redhatpyxis/redhatpyxis.go@@ -0,0 +1,132 @@+package redhatpyxis++import (+ "context"+ "fmt"+ "io"+ "net/http"++ regexp "github.com/wasilibs/go-re2"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"+)++type Scanner struct {+ client *http.Client+}++// Compile-time interface check+var _ detectors.Detector = (*Scanner)(nil)++var (+ defaultClient = detectors.NewClientWithDedup(common.SaneHttpClient())++ pyxisAPIKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"redhat", "pyxis"}) + `\b([a-z0-9]{32})\b`)+)++// Keywords used for fast pre-filtering+func (s Scanner) Keywords() []string {+ return []string{+ "redhat",+ "pyxis",+ }+}++func (s Scanner) getClient() *http.Client {+ if s.client != nil {+ return s.client+ }+ return defaultClient+}++// FromData scans for Red Hat Pyxis API keys and optionally verifies them.+func (s Scanner) FromData(+ ctx context.Context,+ verify bool,+ data []byte,+) (results []detectors.Result, err error) {++ dataStr := string(data)++ uniqueTokens := make(map[string]struct{})+ for _, match := range pyxisAPIKeyPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueTokens[match[1]] = struct{}{}+ }++ for token := range uniqueTokens {+ result := detectors.Result{+ DetectorType: detector_typepb.DetectorType_RedHatPyxis,+ Raw: []byte(token),+ SecretParts: map[string]string{+ "key": token,+ },+ }++ if verify {+ verified, verificationErr := verifyPyxisAPIKey(+ ctx,+ s.getClient(),+ token,+ )+ result.SetVerificationError(verificationErr, token)+ result.Verified = verified+ }++ results = append(results, result)+ }++ return+}++func verifyPyxisAPIKey(+ ctx context.Context,+ client *http.Client,+ token string,+) (bool, error) {++ req, err := http.NewRequestWithContext(+ ctx,+ http.MethodGet,+ "https://catalog.redhat.com/api/containers/v1/projects/certification/requests/images?page_size=1&page=0",+ http.NoBody,+ )+ if err != nil {+ return false, err+ }++ req.Header.Set("X-API-KEY", token)++ res, err := detectors.DoWithDedup(client, detector_typepb.DetectorType_RedHatPyxis, token, req)+ if err != nil {+ return false, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()++ switch res.StatusCode {+ case http.StatusOK:+ return true, nil++ case http.StatusUnauthorized:+ // Invalid API key+ return false, nil++ default:+ return false, fmt.Errorf(+ "unexpected HTTP response status %d",+ res.StatusCode,+ )+ }+}++func (s Scanner) Type() detector_typepb.DetectorType {+ return detector_typepb.DetectorType_RedHatPyxis+}++func (s Scanner) Description() string {+ return "Red Hat Pyxis is a container certification and metadata management platform. Red Hat Pyxis API keys can be used to authenticate against the Red Hat Ecosystem Catalog APIs and access container certification resources and metadata."+}diff --git a/pkg/detectors/redhatpyxis/redhatpyxis_integration_test.go b/pkg/detectors/redhatpyxis/redhatpyxis_integration_test.gonew file mode 100644index 000000000..395355eb8--- /dev/null+++ b/pkg/detectors/redhatpyxis/redhatpyxis_integration_test.go@@ -0,0 +1,211 @@+//go:build detectors+// +build detectors++package redhatpyxis++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"+)++func TestRedHatPyxis_FromData(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)+ defer cancel()++ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors6")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }++ activeToken := testSecrets.MustGetField("REDHAT_PYXIS_API_KEY")++ // Random inactive token matching the expected format+ inactiveToken := "o9ynnj1wfw33a50g9009ti0ne1kqe8ac"++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }++ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: fmt.Appendf(+ []byte{},+ "redhat api key %s",+ activeToken,+ ),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detector_typepb.DetectorType_RedHatPyxis,+ Verified: true,+ Raw: []byte(activeToken),+ },+ },+ },+ {+ name: "found, real token, verification error due to timeout",+ s: Scanner{+ client: common.SaneHttpClientTimeOut(1 * time.Microsecond),+ },+ args: args{+ ctx: context.Background(),+ data: fmt.Appendf(+ []byte{},+ "redhat api key %s",+ activeToken,+ ),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detector_typepb.DetectorType_RedHatPyxis,+ Verified: false,+ Raw: []byte(activeToken),+ },+ },+ wantVerificationErr: true,+ },+ {+ name: "found, real token, verification error due to unexpected api surface",+ s: Scanner{+ client: common.ConstantResponseHttpClient(500, "{}"),+ },+ args: args{+ ctx: context.Background(),+ data: fmt.Appendf(+ []byte{},+ "redhat api key %s",+ activeToken,+ ),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detector_typepb.DetectorType_RedHatPyxis,+ Verified: false,+ Raw: []byte(activeToken),+ },+ },+ wantVerificationErr: true,+ },+ {+ name: "found, unverified (inactive token)",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: fmt.Appendf(+ []byte{},+ "redhat api key %s",+ inactiveToken,+ ),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detector_typepb.DetectorType_RedHatPyxis,+ Verified: false,+ Raw: []byte(inactiveToken),+ },+ },+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("no secrets here"),+ verify: true,+ },+ want: nil,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(+ tt.args.ctx,+ tt.args.verify,+ tt.args.data,+ )++ if (err != nil) != tt.wantErr {+ t.Fatalf(+ "RedHatPyxis.FromData() error = %v, wantErr %v",+ err,+ tt.wantErr,+ )+ }++ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatal("no raw secret present")+ }++ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf(+ "wantVerificationError = %v, verification error = %v",+ tt.wantVerificationErr,+ got[i].VerificationError(),+ )+ }+ }++ ignoreOpts := cmpopts.IgnoreFields(+ detectors.Result{},+ "ExtraData",+ "verificationError",+ "primarySecret",+ "SecretParts",+ )++ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf(+ "RedHatPyxis.FromData() %s diff: (-got +want)\n%s",+ tt.name,+ diff,+ )+ }+ })+ }+}++func BenchmarkRedHatPyxis_FromData(b *testing.B) {+ ctx := context.Background()+ s := Scanner{}++ for name, data := range detectors.MustGetBenchmarkData() {+ b.Run(name, func(b *testing.B) {+ b.ResetTimer()++ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/redhatpyxis/redhatpyxis_test.go b/pkg/detectors/redhatpyxis/redhatpyxis_test.gonew file mode 100644index 000000000..3bd14cc9d--- /dev/null+++ b/pkg/detectors/redhatpyxis/redhatpyxis_test.go@@ -0,0 +1,129 @@+package redhatpyxis++import (+ "context"+ "testing"++ "github.com/google/go-cmp/cmp"+ "github.com/stretchr/testify/require"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++func TestRedHatPyxis_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})++ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "valid pattern - basic",+ input: `+ [INFO] redhat integration initialized+ [DEBUG] redhat_api_key=o9ynnj1wfw33a50g9009ti0ne1kqe8ac+ `,+ want: []string{+ "o9ynnj1wfw33a50g9009ti0ne1kqe8ac",+ },+ },+ {+ name: "valid pattern - multiple tokens",+ input: `+ redhat nvropikx1e0vzxpad8upkf8696icmidx+ redhatpyxis u0oxtxaurp1lbsagehya1rk6r9cxcc7f+ `,+ want: []string{+ "nvropikx1e0vzxpad8upkf8696icmidx",+ "u0oxtxaurp1lbsagehya1rk6r9cxcc7f",+ },+ },+ {+ name: "invalid pattern - uppercase characters",+ input: `+ redhat O9YNNJ1WFW33A50G9009TI0NE1KQE8AC+ `,+ want: nil,+ },+ {+ name: "invalid pattern - too short",+ input: `+ redhat abc123+ `,+ want: nil,+ },+ {+ name: "invalid pattern - too long",+ input: `+ redhat o9ynnj1wfw33a50g9009ti0ne1kqe8acabc+ `,+ want: nil,+ },+ {+ name: "invalid pattern - no keyword nearby",+ input: `+ o9ynnj1wfw33a50g9009ti0ne1kqe8ac+ `,+ want: nil,+ },+ {+ name: "invalid pattern - keyword only",+ input: `+ redhat api key configured+ `,+ want: nil,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))++ if len(test.want) > 0 && len(matchedDetectors) == 0 {+ t.Errorf(+ "test %q failed: expected keywords %v to be found in the input",+ test.name,+ d.Keywords(),+ )+ return+ }++ results, err := d.FromData(+ context.Background(),+ false,+ []byte(test.input),+ )+ require.NoError(t, err)++ if len(results) != len(test.want) {+ t.Errorf(+ "mismatch in result count: expected %d, got %d",+ len(test.want),+ len(results),+ )+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }++ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.goindex 42340b9f3..37c3be814 100644--- a/pkg/engine/defaults/defaults.go+++ b/pkg/engine/defaults/defaults.go@@ -620,6 +620,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/reallysimplesystems""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rebrandly""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rechargepayments"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redhatpyxis""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redis""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/refiner""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rentman"@@ -1520,6 +1521,7 @@ func buildDetectorList() []detectors.Detector {&reallysimplesystems.Scanner{},&rebrandly.Scanner{},&rechargepayments.Scanner{},+ &redhatpyxis.Scanner{},&redis.Scanner{},&refiner.Scanner{},&rentman.Scanner{},@@ -1808,6 +1810,8 @@ func buildDetectorList() []detectors.Detector {return !feature.BraintrustDetectorEnabled.Load()case *pganalyzereadkey.Scanner:return !feature.PgAnalyzeReadKeyDetectorEnabled.Load()+ case *redhatpyxis.Scanner:+ return !feature.RedHatPyxisDetectorEnabled.Load()default:return false}diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.goindex d7bb7f9f8..c46367092 100644--- a/pkg/engine/defaults/defaults_test.go+++ b/pkg/engine/defaults/defaults_test.go@@ -136,6 +136,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{detector_typepb.DetectorType_User: {},detector_typepb.DetectorType_BrainTrustApiKey: {},detector_typepb.DetectorType_PgAnalyzeReadKey: {},+ detector_typepb.DetectorType_RedHatPyxis: {},// Reserved / special types.detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()diff --git a/pkg/feature/feature.go b/pkg/feature/feature.goindex adb865531..d053e30e6 100644--- a/pkg/feature/feature.go+++ b/pkg/feature/feature.go@@ -27,6 +27,7 @@ var (UserDetectorEnabled atomic.BoolBraintrustDetectorEnabled atomic.BoolPgAnalyzeReadKeyDetectorEnabled atomic.Bool+ RedHatPyxisDetectorEnabled atomic.Bool)type AtomicString struct {diff --git a/pkg/pb/detector_typepb/detector_type.pb.go b/pkg/pb/detector_typepb/detector_type.pb.goindex 0e886cef4..7d46f6422 100644--- a/pkg/pb/detector_typepb/detector_type.pb.go+++ b/pkg/pb/detector_typepb/detector_type.pb.go@@ -1108,6 +1108,7 @@ const (DetectorType_AWSAppSync DetectorType = 1052DetectorType_BrainTrustApiKey DetectorType = 1053DetectorType_PgAnalyzeReadKey DetectorType = 1054+ DetectorType_RedHatPyxis DetectorType = 1055)// Enum value maps for DetectorType.@@ -2164,6 +2165,7 @@ var (1052: "AWSAppSync",1053: "BrainTrustApiKey",1054: "PgAnalyzeReadKey",+ 1055: "RedHatPyxis",}DetectorType_value = map[string]int32{"Alibaba": 0,@@ -3217,6 +3219,7 @@ var ("AWSAppSync": 1052,"BrainTrustApiKey": 1053,"PgAnalyzeReadKey": 1054,+ "RedHatPyxis": 1055,})@@ -3252,7 +3255,7 @@ var File_detector_type_proto protoreflect.FileDescriptorvar file_detector_type_proto_rawDesc = []byte{0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e,0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f,- 0x74, 0x79, 0x70, 0x65, 0x2a, 0x90, 0x89, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,+ 0x74, 0x79, 0x70, 0x65, 0x2a, 0xa2, 0x89, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62,0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a,0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10,@@ -4349,12 +4352,13 @@ var file_detector_type_proto_rawDesc = []byte{0x41, 0x70, 0x70, 0x53, 0x79, 0x6e, 0x63, 0x10, 0x9c, 0x08, 0x12, 0x15, 0x0a, 0x10, 0x42, 0x72,0x61, 0x69, 0x6e, 0x54, 0x72, 0x75, 0x73, 0x74, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0x9d,0x08, 0x12, 0x15, 0x0a, 0x10, 0x50, 0x67, 0x41, 0x6e, 0x61, 0x6c, 0x79, 0x7a, 0x65, 0x52, 0x65,- 0x61, 0x64, 0x4b, 0x65, 0x79, 0x10, 0x9e, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68,- 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,- 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,- 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,- 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,- 0x74, 0x6f, 0x33,+ 0x61, 0x64, 0x4b, 0x65, 0x79, 0x10, 0x9e, 0x08, 0x12, 0x10, 0x0a, 0x0b, 0x52, 0x65, 0x64, 0x48,+ 0x61, 0x74, 0x50, 0x79, 0x78, 0x69, 0x73, 0x10, 0x9f, 0x08, 0x42, 0x41, 0x5a, 0x3f, 0x67, 0x69,+ 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,+ 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65,+ 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65,+ 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x70, 0x62, 0x62, 0x06, 0x70,+ 0x72, 0x6f, 0x74, 0x6f, 0x33,}var (diff --git a/proto/detector_type.proto b/proto/detector_type.protoindex d5d2db4d4..ebbeb3f33 100644--- a/proto/detector_type.proto+++ b/proto/detector_type.proto@@ -1056,4 +1056,5 @@ enum DetectorType {AWSAppSync = 1052;BrainTrustApiKey = 1053;PgAnalyzeReadKey = 1054;+ RedHatPyxis = 1055;}