Skip to content

Find, verify, and analyze leaked credentials

Refactored Gitlab Detectors (#4323)

Kashif Khan761dcfa94cd3
diff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.go
index c3f6f41e6..a83b42bc5 100644
--- a/pkg/detectors/gitlab/v1/gitlab.go
+++ b/pkg/detectors/gitlab/v1/gitlab.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "maps"
"net/http"
"strings"
@@ -38,12 +39,28 @@ var (
BlockedUserMessage = "403 Forbidden - Your account has been blocked"
)
+func (s Scanner) getClient() *http.Client {
+ if s.client != nil {
+ return s.client
+ }
+
+ return defaultClient
+}
+
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"gitlab"}
}
+func (s Scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_Gitlab
+}
+
+func (s Scanner) Description() string {
+ return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab API tokens can be used to access and modify repository data and other resources."
+}
+
// FromData will find and optionally verify Gitlab secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -72,14 +89,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)
- s1.Verified = isVerified
- for key, value := range extraData {
- s1.ExtraData[key] = value
+ for _, endpoint := range s.Endpoints() {
+ isVerified, extraData, verificationErr := VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
+ s1.Verified = isVerified
+ maps.Copy(s1.ExtraData, extraData)
+
+ s1.SetVerificationError(verificationErr)
+
+ // for verified keys set the analysis info
+ if s1.Verified {
+ s1.AnalysisInfo = map[string]string{
+ "key": resMatch,
+ "host": endpoint,
+ }
+ }
}
-
- s1.SetVerificationError(verificationErr, resMatch)
- s1.AnalysisInfo = analysisInfo
}
results = append(results, s1)
@@ -88,74 +112,51 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {
+func VerifyGitlab(ctx context.Context, client *http.Client, baseEndpoint, resMatch string) (bool, map[string]string, error) {
// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
// of them contain data, we know we have a valid key, but if they all fail, we don't
- client := s.client
- if client == nil {
- client = defaultClient
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseEndpoint+"/api/v4/user", http.NoBody)
+ if err != nil {
+ return false, nil, err
}
- for _, baseURL := range s.Endpoints() {
- // test `read_user` scope
- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
- if err != nil {
- continue
- }
-
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
- res, err := client.Do(req)
- if err != nil {
- return false, nil, nil, err
- }
-
- defer res.Body.Close()
- bodyBytes, err := io.ReadAll(res.Body)
- if err != nil {
- return false, nil, nil, err
- }
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+ res, err := client.Do(req)
+ if err != nil {
+ return false, nil, err
+ }
- analysisInfo := map[string]string{
- "key": resMatch,
- "host": baseURL,
- }
+ defer res.Body.Close()
- // 200 means good key and has `read_user` scope
- // 403 means good key but not the right scope
- // 401 is bad key
- switch res.StatusCode {
- case http.StatusOK:
- return json.Valid(bodyBytes), nil, analysisInfo, nil
- case http.StatusForbidden:
- // check if the user account is blocked or not
- stringBody := string(bodyBytes)
- if strings.Contains(stringBody, BlockedUserMessage) {
- return true, map[string]string{
- "blocked": "True",
- }, analysisInfo, nil
- }
+ bodyBytes, err := io.ReadAll(res.Body)
+ if err != nil {
+ return false, nil, err
+ }
- // Good key but not the right scope
- return true, nil, analysisInfo, nil
- case http.StatusUnauthorized:
- // Nothing to do; zero values are the ones we want
- return false, nil, nil, nil
- default:
- return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+ // 200 means good key and has `read_user` scope
+ // 403 means good key but not the right scope
+ // 401 is bad key
+ switch res.StatusCode {
+ case http.StatusOK:
+ return json.Valid(bodyBytes), nil, nil
+ case http.StatusForbidden:
+ // check if the user account is blocked or not
+ stringBody := string(bodyBytes)
+ if strings.Contains(stringBody, BlockedUserMessage) {
+ return true, map[string]string{
+ "blocked": "True",
+ }, nil
}
+ // Good key but not the right scope
+ return true, nil, nil
+ case http.StatusUnauthorized:
+ // Nothing to do; zero values are the ones we want
+ return false, nil, nil
+ default:
+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
-
- return false, nil, nil, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
- return detectorspb.DetectorType_Gitlab
-}
-
-func (s Scanner) Description() string {
- return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab API tokens can be used to access and modify repository data and other resources."
}
diff --git a/pkg/detectors/gitlab/v1/gitlab_integration_test.go b/pkg/detectors/gitlab/v1/gitlab_integration_test.go
index f9e923041..0b3d7bd5a 100644
--- a/pkg/detectors/gitlab/v1/gitlab_integration_test.go
+++ b/pkg/detectors/gitlab/v1/gitlab_integration_test.go
@@ -189,7 +189,7 @@ func TestGitlab_FromChunk(t *testing.T) {
}
got[i].AnalysisInfo = nil
}
- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
diff --git a/pkg/detectors/gitlab/v2/gitlab_integration_test.go b/pkg/detectors/gitlab/v2/gitlab_integration_test.go
index f10683d84..285119298 100644
--- a/pkg/detectors/gitlab/v2/gitlab_integration_test.go
+++ b/pkg/detectors/gitlab/v2/gitlab_integration_test.go
@@ -101,7 +101,7 @@ func TestGitlabV2_FromChunk_WithV1Secrets(t *testing.T) {
}
got[i].AnalysisInfo = nil
}
- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
@@ -280,7 +280,7 @@ func TestGitlabV2_FromChunk(t *testing.T) {
}
got[i].AnalysisInfo = nil
}
- opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError")
+ opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "verificationError", "primarySecret")
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
diff --git a/pkg/detectors/gitlab/v2/gitlab_v2.go b/pkg/detectors/gitlab/v2/gitlab_v2.go
index 30f42e181..284fc9f8b 100644
--- a/pkg/detectors/gitlab/v2/gitlab_v2.go
+++ b/pkg/detectors/gitlab/v2/gitlab_v2.go
@@ -3,7 +3,7 @@ package gitlab
import (
"context"
"fmt"
- "io"
+ "maps"
"net/http"
"strings"
@@ -33,10 +33,26 @@ var (
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
)
+func (s Scanner) getClient() *http.Client {
+ if s.client != nil {
+ return s.client
+ }
+
+ return defaultClient
+}
+
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string { return []string{"glpat-"} }
+func (s Scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_Gitlab
+}
+
+func (s Scanner) Description() string {
+ return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."
+}
+
// FromData will find and optionally verify Gitlab secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -56,14 +72,21 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)
- s1.Verified = isVerified
- for key, value := range extraData {
- s1.ExtraData[key] = value
+ for _, endpoint := range s.Endpoints() {
+ isVerified, extraData, verificationErr := v1.VerifyGitlab(ctx, s.getClient(), endpoint, resMatch)
+ s1.Verified = isVerified
+ maps.Copy(s1.ExtraData, extraData)
+
+ s1.SetVerificationError(verificationErr)
+
+ // for verified keys set the analysis info
+ if s1.Verified {
+ s1.AnalysisInfo = map[string]string{
+ "key": resMatch,
+ "host": endpoint,
+ }
+ }
}
-
- s1.SetVerificationError(verificationErr, resMatch)
- s1.AnalysisInfo = analysisInfo
}
results = append(results, s1)
@@ -71,72 +94,3 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
-
-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {
- // there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry
- // they all grant access to different parts of the API. I couldn't find an endpoint that every
- // one of these scopes has access to, so we just check an example endpoint for each scope. If any
- // of them contain data, we know we have a valid key, but if they all fail, we don't
-
- client := s.client
- if client == nil {
- client = defaultClient
- }
- for _, baseURL := range s.Endpoints() {
- // test `read_user` scope
- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
- if err != nil {
- continue
- }
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
- res, err := client.Do(req)
- if err != nil {
- return false, nil, nil, err
- }
- defer res.Body.Close()
-
- bodyBytes, err := io.ReadAll(res.Body)
- if err != nil {
- return false, nil, nil, err
- }
-
- analysisInfo := map[string]string{
- "key": resMatch,
- "host": baseURL,
- }
-
- // 200 means good key and has `read_user` scope
- // 403 means good key but not the right scope
- // 401 is bad key
- switch res.StatusCode {
- case http.StatusOK:
- return true, nil, analysisInfo, nil
- case http.StatusForbidden:
- // check if the user account is blocked or not
- stringBody := string(bodyBytes)
- if strings.Contains(stringBody, v1.BlockedUserMessage) {
- return true, map[string]string{
- "blocked": "True",
- }, analysisInfo, nil
- }
-
- // Good key but not the right scope
- return true, nil, analysisInfo, nil
- case http.StatusUnauthorized:
- // Nothing to do; zero values are the ones we want
- return false, nil, nil, nil
- default:
- return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
- }
-
- }
- return false, nil, nil, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
- return detectorspb.DetectorType_Gitlab
-}
-
-func (s Scanner) Description() string {
- return "GitLab is a web-based DevOps lifecycle tool that provides a Git repository manager providing wiki, issue-tracking, and CI/CD pipeline features. GitLab Personal Access Tokens (PATs) can be used to authenticate and access GitLab resources."
-}