Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fixed gitlab detector (#4371)
Kashif Khan836ed28cc7fc
diff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.goindex a83b42bc5..fa3c63b84 100644--- a/pkg/detectors/gitlab/v1/gitlab.go+++ b/pkg/detectors/gitlab/v1/gitlab.go@@ -67,29 +67,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultmatches := keyPat.FindAllStringSubmatch(dataStr, -1)for _, match := range matches {+ resMatch := strings.TrimSpace(match[1])+// ignore v2 detectors which have a prefix of `glpat-`if strings.Contains(match[0], "glpat-") {continue}- resMatch := strings.TrimSpace(match[1])// to avoid false positivesif detectors.StringShannonEntropy(resMatch) < 3.6 {continue}- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_Gitlab,- Raw: []byte(resMatch),- ExtraData: map[string]string{},- }- s1.ExtraData = map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",- "version": fmt.Sprintf("%d", s.Version()),- }+ for _, endpoint := range s.Endpoints() {+ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Gitlab,+ Raw: []byte(resMatch),+ RawV2: []byte(resMatch + endpoint),+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",+ "version": fmt.Sprintf("%d", s.Version()),+ },+ }- if verify {- for _, endpoint := range s.Endpoints() {+ if verify {isVerified, extraData, verificationErr := VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)s1.Verified = isVerifiedmaps.Copy(s1.ExtraData, extraData)@@ -102,11 +103,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result"key": resMatch,"host": endpoint,}++ // if secret is verified with one endpoint, break the loop to continue to next secret+ results = append(results, s1)+ break}}- }- results = append(results, s1)+ results = append(results, s1)+ }}return results, nildiff --git a/pkg/detectors/gitlab/v1/gitlab_integration_test.go b/pkg/detectors/gitlab/v1/gitlab_integration_test.goindex 0b3d7bd5a..8c71aee80 100644--- a/pkg/detectors/gitlab/v1/gitlab_integration_test.go+++ b/pkg/detectors/gitlab/v1/gitlab_integration_test.go@@ -175,6 +175,8 @@ func TestGitlab_FromChunk(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {+ tt.s.SetCloudEndpoint("https://gitlab.com")+ tt.s.UseCloudEndpoint(true)got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)if (err != nil) != tt.wantErr {t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)@@ -189,7 +191,7 @@ func TestGitlab_FromChunk(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}@@ -287,7 +289,7 @@ func TestGitlab_FromChunk_WithV2Secrets(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/gitlab/v1/gitlab_v1_test.go b/pkg/detectors/gitlab/v1/gitlab_v1_test.goindex b093f22d8..b5d479bf7 100644--- a/pkg/detectors/gitlab/v1/gitlab_v1_test.go+++ b/pkg/detectors/gitlab/v1/gitlab_v1_test.go@@ -10,29 +10,10 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-var (- validPattern = `[{- "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",- "name": "Gitlab",- "type": "Detector",- "api": true,- "authentication_type": "",- "verification_url": "https://api.example.com/example",- "test_secrets": {- "gitlab_secret": "oXCt4JT2wf1_WlZl2OVG"- },- "docs":"https://docs.gitlab.com/test/api/example.json#get-drone-test-example-settings", // this matches the pattern but fail in entropy check- "expected_response": "200",- "method": "GET",- "deprecated": false- }]`- secret = "oXCt4JT2wf1_WlZl2OVG"- validPattern2 = "GITLAB_TOKEN=ABc123456789dEFghIJK"- secret2 = "ABc123456789dEFghIJK"-)-func TestGitLab_Pattern(t *testing.T) {d := Scanner{}+ d.SetCloudEndpoint("https://gitlab.com")+ d.UseCloudEndpoint(true)ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {@@ -41,14 +22,28 @@ func TestGitLab_Pattern(t *testing.T) {want []string}{{- name: "valid pattern",- input: validPattern,- want: []string{secret},+ name: "valid pattern",+ input: `[{+ "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",+ "name": "Gitlab",+ "type": "Detector",+ "api": true,+ "authentication_type": "",+ "verification_url": "https://api.example.com/example",+ "test_secrets": {+ "gitlab_secret": "oXCt4JT2wf1_WlZl2OVG"+ },+ "docs":"https://docs.gitlab.com/test/api/example.json#get-drone-test-example-settings", // this matches the pattern but fail in entropy check+ "expected_response": "200",+ "method": "GET",+ "deprecated": false+ }]`,+ want: []string{"oXCt4JT2wf1_WlZl2OVGhttps://gitlab.com"},},{name: "valid pattern (with = before secret)",- input: validPattern2,- want: []string{secret2},+ input: "GITLAB_TOKEN=ABc123456789dEFghIJK",+ want: []string{"ABc123456789dEFghIJKhttps://gitlab.com"},},}diff --git a/pkg/detectors/gitlab/v2/gitlab_integration_test.go b/pkg/detectors/gitlab/v2/gitlab_integration_test.goindex 285119298..88e86040b 100644--- a/pkg/detectors/gitlab/v2/gitlab_integration_test.go+++ b/pkg/detectors/gitlab/v2/gitlab_integration_test.go@@ -87,6 +87,8 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {+ tt.s.SetCloudEndpoint("https://gitlab.com")+ tt.s.UseCloudEndpoint(true)got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)if (err != nil) != tt.wantErr {t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)@@ -101,7 +103,7 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}@@ -280,7 +282,7 @@ func TestGitlabV2_FromChunk(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/gitlab/v2/gitlab_v2.go b/pkg/detectors/gitlab/v2/gitlab_v2.goindex 284fc9f8b..753f63e84 100644--- a/pkg/detectors/gitlab/v2/gitlab_v2.go+++ b/pkg/detectors/gitlab/v2/gitlab_v2.go@@ -59,20 +59,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultmatches := keyPat.FindAllStringSubmatch(dataStr, -1)for _, match := range matches {-resMatch := strings.TrimSpace(match[1])- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_Gitlab,- Raw: []byte(resMatch),- ExtraData: map[string]string{},- }- s1.ExtraData = map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",- "version": fmt.Sprintf("%d", s.Version()),- }- if verify {- for _, endpoint := range s.Endpoints() {+ for _, endpoint := range s.Endpoints() {+ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_Gitlab,+ Raw: []byte(resMatch),+ RawV2: []byte(resMatch + endpoint),+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/gitlab/",+ "version": fmt.Sprintf("%d", s.Version()),+ },+ }++ if verify {+isVerified, extraData, verificationErr := v1.VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)s1.Verified = isVerifiedmaps.Copy(s1.ExtraData, extraData)@@ -85,11 +86,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result"key": resMatch,"host": endpoint,}++ // if secret is verified with one endpoint, break the loop to continue to next secret+ results = append(results, s1)+ break}}- }- results = append(results, s1)+ results = append(results, s1)+ }}return results, nildiff --git a/pkg/detectors/gitlab/v2/gitlab_v2_test.go b/pkg/detectors/gitlab/v2/gitlab_v2_test.goindex e686a37ee..05f8e815f 100644--- a/pkg/detectors/gitlab/v2/gitlab_v2_test.go+++ b/pkg/detectors/gitlab/v2/gitlab_v2_test.go@@ -10,26 +10,10 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-var (- validPattern = `[{- "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",- "name": "Gitlab",- "type": "Detector",- "api": true,- "authentication_type": "",- "verification_url": "https://api.example.com/example",- "test_secrets": {- "gitlab_secret": "glpat-W6fYSu70dPEo5w_SwbHWgQ"- },- "expected_response": "200",- "method": "GET",- "deprecated": false- }]`- secret = "glpat-W6fYSu70dPEo5w_SwbHWgQ"-)-func TestGitLab_Pattern(t *testing.T) {d := Scanner{}+ d.SetCloudEndpoint("https://gitlab.com")+ d.UseCloudEndpoint(true)ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {@@ -38,9 +22,22 @@ func TestGitLab_Pattern(t *testing.T) {want []string}{{- name: "valid pattern",- input: validPattern,- want: []string{secret},+ name: "valid pattern",+ input: `[{+ "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",+ "name": "Gitlab",+ "type": "Detector",+ "api": true,+ "authentication_type": "",+ "verification_url": "https://api.example.com/example",+ "test_secrets": {+ "gitlab_secret": "glpat-W6fYSu70dPEo5w_SwbHWgQ"+ },+ "expected_response": "200",+ "method": "GET",+ "deprecated": false+ }]`,+ want: []string{"glpat-W6fYSu70dPEo5w_SwbHWgQhttps://gitlab.com"},},}