Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Refactored Gitlab Detectors (#4323)
Kashif Khan761dcfa94cd3
diff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.goindex c3f6f41e6..a83b42bc5 100644--- a/pkg/detectors/gitlab/v1/gitlab.go+++ b/pkg/detectors/gitlab/v1/gitlab.go@@ -5,6 +5,7 @@ import ("encoding/json""fmt""io"+ "maps""net/http""strings"@@ -38,12 +39,28 @@ var (BlockedUserMessage = "403 Forbidden - Your account has been blocked")+func (s Scanner) getClient() *http.Client {+ if s.client != nil {+ return s.client+ }++ return defaultClient+}+// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {return []string{"gitlab"}}+func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_Gitlab+}++func (s Scanner) Description() string {+ return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab API tokens can be used to access and modify repository data and other resources."+}+// FromData will find and optionally verify Gitlab secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)@@ -72,14 +89,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)- s1.Verified = isVerified- for key, value := range extraData {- s1.ExtraData[key] = value+ for _, endpoint := range s.Endpoints() {+ isVerified, extraData, verificationErr := VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)+ s1.Verified = isVerified+ maps.Copy(s1.ExtraData, extraData)++ s1.SetVerificationError(verificationErr)++ // for verified keys set the analysis info+ if s1.Verified {+ s1.AnalysisInfo = map[string]string{+ "key": resMatch,+ "host": endpoint,+ }+ }}-- s1.SetVerificationError(verificationErr, resMatch)- s1.AnalysisInfo = analysisInfo}results = append(results, s1)@@ -88,74 +112,51 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {+func VerifyGitlab(ctx context.Context, client *http.Client, baseEndpoint, resMatch string) (bool, map[string]string, error) {// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry// they all grant access to different parts of the API. I couldn't find an endpoint that every// one of these scopes has access to, so we just check an example endpoint for each scope. If any// of them contain data, we know we have a valid key, but if they all fail, we don't- client := s.client- if client == nil {- client = defaultClient+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseEndpoint+"/api/v4/user", http.NoBody)+ if err != nil {+ return false, nil, err}- for _, baseURL := range s.Endpoints() {- // test `read_user` scope- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)- if err != nil {- continue- }-- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))- res, err := client.Do(req)- if err != nil {- return false, nil, nil, err- }-- defer res.Body.Close()- bodyBytes, err := io.ReadAll(res.Body)- if err != nil {- return false, nil, nil, err- }+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))+ res, err := client.Do(req)+ if err != nil {+ return false, nil, err+ }- analysisInfo := map[string]string{- "key": resMatch,- "host": baseURL,- }+ defer res.Body.Close()- // 200 means good key and has `read_user` scope- // 403 means good key but not the right scope- // 401 is bad key- switch res.StatusCode {- case http.StatusOK:- return json.Valid(bodyBytes), nil, analysisInfo, nil- case http.StatusForbidden:- // check if the user account is blocked or not- stringBody := string(bodyBytes)- if strings.Contains(stringBody, BlockedUserMessage) {- return true, map[string]string{- "blocked": "True",- }, analysisInfo, nil- }+ bodyBytes, err := io.ReadAll(res.Body)+ if err != nil {+ return false, nil, err+ }- // Good key but not the right scope- return true, nil, analysisInfo, nil- case http.StatusUnauthorized:- // Nothing to do; zero values are the ones we want- return false, nil, nil, nil- default:- return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ // 200 means good key and has `read_user` scope+ // 403 means good key but not the right scope+ // 401 is bad key+ switch res.StatusCode {+ case http.StatusOK:+ return json.Valid(bodyBytes), nil, nil+ case http.StatusForbidden:+ // check if the user account is blocked or not+ stringBody := string(bodyBytes)+ if strings.Contains(stringBody, BlockedUserMessage) {+ return true, map[string]string{+ "blocked": "True",+ }, nil}+ // Good key but not the right scope+ return true, nil, nil+ case http.StatusUnauthorized:+ // Nothing to do; zero values are the ones we want+ return false, nil, nil+ default:+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)}-- return false, nil, nil, nil-}--func (s Scanner) Type() detectorspb.DetectorType {- return detectorspb.DetectorType_Gitlab-}--func (s Scanner) Description() string {- return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab API tokens can be used to access and modify repository data and other resources."}diff --git a/pkg/detectors/gitlab/v1/gitlab_integration_test.go b/pkg/detectors/gitlab/v1/gitlab_integration_test.goindex f9e923041..0b3d7bd5a 100644--- a/pkg/detectors/gitlab/v1/gitlab_integration_test.go+++ b/pkg/detectors/gitlab/v1/gitlab_integration_test.go@@ -189,7 +189,7 @@ func TestGitlab_FromChunk(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/gitlab/v2/gitlab_integration_test.go b/pkg/detectors/gitlab/v2/gitlab_integration_test.goindex f10683d84..285119298 100644--- a/pkg/detectors/gitlab/v2/gitlab_integration_test.go+++ b/pkg/detectors/gitlab/v2/gitlab_integration_test.go@@ -101,7 +101,7 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}@@ -280,7 +280,7 @@ func TestGitlabV2_FromChunk(t *testing.T) {}got[i].AnalysisInfo = nil}- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, opts); diff != "" {t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/gitlab/v2/gitlab_v2.go b/pkg/detectors/gitlab/v2/gitlab_v2.goindex 30f42e181..284fc9f8b 100644--- a/pkg/detectors/gitlab/v2/gitlab_v2.go+++ b/pkg/detectors/gitlab/v2/gitlab_v2.go@@ -3,7 +3,7 @@ package gitlabimport ("context""fmt"- "io"+ "maps""net/http""strings"@@ -33,10 +33,26 @@ var (keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`))+func (s Scanner) getClient() *http.Client {+ if s.client != nil {+ return s.client+ }++ return defaultClient+}+// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string { return []string{"glpat-"} }+func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_Gitlab+}++func (s Scanner) Description() string {+ return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."+}+// FromData will find and optionally verify Gitlab secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)@@ -56,14 +72,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)- s1.Verified = isVerified- for key, value := range extraData {- s1.ExtraData[key] = value+ for _, endpoint := range s.Endpoints() {+ isVerified, extraData, verificationErr := v1.VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)+ s1.Verified = isVerified+ maps.Copy(s1.ExtraData, extraData)++ s1.SetVerificationError(verificationErr)++ // for verified keys set the analysis info+ if s1.Verified {+ s1.AnalysisInfo = map[string]string{+ "key": resMatch,+ "host": endpoint,+ }+ }}-- s1.SetVerificationError(verificationErr, resMatch)- s1.AnalysisInfo = analysisInfo}results = append(results, s1)@@ -71,72 +94,3 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}--func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {- // there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry- // they all grant access to different parts of the API. I couldn't find an endpoint that every- // one of these scopes has access to, so we just check an example endpoint for each scope. If any- // of them contain data, we know we have a valid key, but if they all fail, we don't-- client := s.client- if client == nil {- client = defaultClient- }- for _, baseURL := range s.Endpoints() {- // test `read_user` scope- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)- if err != nil {- continue- }- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))- res, err := client.Do(req)- if err != nil {- return false, nil, nil, err- }- defer res.Body.Close()-- bodyBytes, err := io.ReadAll(res.Body)- if err != nil {- return false, nil, nil, err- }-- analysisInfo := map[string]string{- "key": resMatch,- "host": baseURL,- }-- // 200 means good key and has `read_user` scope- // 403 means good key but not the right scope- // 401 is bad key- switch res.StatusCode {- case http.StatusOK:- return true, nil, analysisInfo, nil- case http.StatusForbidden:- // check if the user account is blocked or not- stringBody := string(bodyBytes)- if strings.Contains(stringBody, v1.BlockedUserMessage) {- return true, map[string]string{- "blocked": "True",- }, analysisInfo, nil- }-- // Good key but not the right scope- return true, nil, analysisInfo, nil- case http.StatusUnauthorized:- // Nothing to do; zero values are the ones we want- return false, nil, nil, nil- default:- return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)- }-- }- return false, nil, nil, nil-}--func (s Scanner) Type() detectorspb.DetectorType {- return detectorspb.DetectorType_Gitlab-}--func (s Scanner) Description() string {- return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."-}