Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Support for Customize Endpoint in Gitlab Analyzer (#3832)
Abdul Basite967efaec90d
diff --git a/pkg/analyzer/analyzers/gitlab/expected_output.json b/pkg/analyzer/analyzers/gitlab/expected_output.jsonindex dd20af460..fe0a8e7c8 100644--- a/pkg/analyzer/analyzers/gitlab/expected_output.json+++ b/pkg/analyzer/analyzers/gitlab/expected_output.json@@ -1 +1 @@-{"AnalyzerType":5,"Bindings":[{"Resource":{"Name":"gitlab.com/user/22466472","FullyQualifiedName":"gitlab.com/user/22466472","Type":"user","Metadata":{"token_created_at":"2024-08-15T06:33:00.337Z","token_expires_at":"2025-08-15","token_id":10470457,"token_name":"test-project-token","token_revoked":false},"Parent":null},"Permission":{"Value":"read_api","Parent":null}},{"Resource":{"Name":"gitlab.com/user/22466472","FullyQualifiedName":"gitlab.com/user/22466472","Type":"user","Metadata":{"token_created_at":"2024-08-15T06:33:00.337Z","token_expires_at":"2025-08-15","token_id":10470457,"token_name":"test-project-token","token_revoked":false},"Parent":null},"Permission":{"Value":"read_repository","Parent":null}},{"Resource":{"Name":"truffletester / trufflehog","FullyQualifiedName":"gitlab.com/project/60871295","Type":"project","Metadata":null,"Parent":null},"Permission":{"Value":"Developer","Parent":null}}],"UnboundedResources":null,"Metadata":{"enterprise":true,"version":"17.6.0-pre"}}\ No newline at end of file+{"AnalyzerType":5,"Bindings":[{"Resource":{"Name":"gitlab.com/user/22466472","FullyQualifiedName":"gitlab.com/user/22466472","Type":"user","Metadata":{"token_created_at":"2024-08-15T06:33:00.337Z","token_expires_at":"2025-08-15","token_id":10470457,"token_name":"test-project-token","token_revoked":false},"Parent":null},"Permission":{"Value":"read_api","Parent":null}},{"Resource":{"Name":"gitlab.com/user/22466472","FullyQualifiedName":"gitlab.com/user/22466472","Type":"user","Metadata":{"token_created_at":"2024-08-15T06:33:00.337Z","token_expires_at":"2025-08-15","token_id":10470457,"token_name":"test-project-token","token_revoked":false},"Parent":null},"Permission":{"Value":"read_repository","Parent":null}},{"Resource":{"Name":"truffletester / trufflehog","FullyQualifiedName":"gitlab.com/project/60871295","Type":"project","Metadata":null,"Parent":null},"Permission":{"Value":"Developer","Parent":null}}],"UnboundedResources":null,"Metadata":{"enterprise":true}}\ No newline at end of filediff --git a/pkg/analyzer/analyzers/gitlab/gitlab.go b/pkg/analyzer/analyzers/gitlab/gitlab.goindex dbfd89119..26383d8e0 100644--- a/pkg/analyzer/analyzers/gitlab/gitlab.go+++ b/pkg/analyzer/analyzers/gitlab/gitlab.go@@ -21,6 +21,10 @@ import (var _ analyzers.Analyzer = (*Analyzer)(nil)+const (+ DefaultGitLabHost = "https://gitlab.com"+)+type Analyzer struct {Cfg *config.Config}@@ -32,8 +36,12 @@ func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyif !ok {return nil, errors.New("key not found in credentialInfo")}+ host, ok := credInfo["host"]+ if !ok {+ host = DefaultGitLabHost+ }- info, err := AnalyzePermissions(a.Cfg, key)+ info, err := AnalyzePermissions(a.Cfg, key, host)if err != nil {return nil, err}@@ -44,7 +52,6 @@ func secretInfoToAnalyzerResult(info *SecretInfo) *analyzers.AnalyzerResult {result := analyzers.AnalyzerResult{AnalyzerType: analyzers.AnalyzerTypeGitLab,Metadata: map[string]any{- "version": info.Metadata.Version,"enterprise": info.Metadata.Enterprise,},Bindings: []analyzers.Binding{},@@ -133,11 +140,11 @@ type MetadataJSON struct {Enterprise bool `json:"enterprise"`}-func getPersonalAccessToken(cfg *config.Config, key string) (AccessTokenJSON, int, error) {+func getPersonalAccessToken(cfg *config.Config, key, host string) (AccessTokenJSON, int, error) {var tokens AccessTokenJSONclient := analyzers.NewAnalyzeClient(cfg)- req, err := http.NewRequest("GET", "https://gitlab.com/api/v4/personal_access_tokens/self", nil)+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/api/v4/personal_access_tokens/self", host), nil)if err != nil {return tokens, -1, err}@@ -155,11 +162,11 @@ func getPersonalAccessToken(cfg *config.Config, key string) (AccessTokenJSON, inreturn tokens, resp.StatusCode, nil}-func getAccessibleProjects(cfg *config.Config, key string) ([]ProjectsJSON, error) {+func getAccessibleProjects(cfg *config.Config, key, host string) ([]ProjectsJSON, error) {var projects []ProjectsJSONclient := analyzers.NewAnalyzeClient(cfg)- req, err := http.NewRequest("GET", "https://gitlab.com/api/v4/projects", nil)+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/api/v4/projects", host), nil)if err != nil {return projects, err}@@ -197,11 +204,11 @@ func getAccessibleProjects(cfg *config.Config, key string) ([]ProjectsJSON, erroreturn projects, nil}-func getMetadata(cfg *config.Config, key string) (MetadataJSON, error) {+func getMetadata(cfg *config.Config, key, host string) (MetadataJSON, error) {var metadata MetadataJSONclient := analyzers.NewAnalyzeClient(cfg)- req, err := http.NewRequest("GET", "https://gitlab.com/api/v4/metadata", nil)+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/api/v4/metadata", host), nil)if err != nil {return metadata, err}@@ -244,9 +251,9 @@ type SecretInfo struct {Projects []ProjectsJSON}-func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) {+func AnalyzePermissions(cfg *config.Config, key string, host string) (*SecretInfo, error) {// get personal_access_tokens accessible- token, statusCode, err := getPersonalAccessToken(cfg, key)+ token, statusCode, err := getPersonalAccessToken(cfg, key, host)if err != nil {return nil, err}@@ -254,12 +261,12 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) {return nil, fmt.Errorf("Invalid GitLab Access Token")}- meta, err := getMetadata(cfg, key)+ meta, err := getMetadata(cfg, key, host)if err != nil {return nil, err}- projects, err := getAccessibleProjects(cfg, key)+ projects, err := getAccessibleProjects(cfg, key, host)if err != nil {return nil, err}@@ -272,7 +279,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) {}func AnalyzeAndPrintPermissions(cfg *config.Config, key string) {- info, err := AnalyzePermissions(cfg, key)+ info, err := AnalyzePermissions(cfg, key, DefaultGitLabHost)if err != nil {color.Red("[x] Error: %s", err)returndiff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.goindex ce0c070f5..301d9fb14 100644--- a/pkg/detectors/gitlab/v1/gitlab.go+++ b/pkg/detectors/gitlab/v1/gitlab.go@@ -70,16 +70,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- isVerified, extraData, verificationErr := s.verifyGitlab(ctx, resMatch)+ isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)s1.Verified = isVerifiedfor key, value := range extraData {s1.ExtraData[key] = value}s1.SetVerificationError(verificationErr, resMatch)- s1.AnalysisInfo = map[string]string{- "key": resMatch,- }+ s1.AnalysisInfo = analysisInfo}results = append(results, s1)@@ -88,7 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, error) {+func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry// they all grant access to different parts of the API. I couldn't find an endpoint that every// one of these scopes has access to, so we just check an example endpoint for each scope. If any@@ -108,14 +106,19 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[sreq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))res, err := client.Do(req)if err != nil {- return false, nil, err+ return false, nil, nil, err}defer res.Body.Close()bodyBytes, err := io.ReadAll(res.Body)if err != nil {- return false, nil, err+ return false, nil, nil, err+ }++ analysisInfo := map[string]string{+ "key": resMatch,+ "host": baseURL,}// 200 means good key and has `read_user` scope@@ -123,28 +126,28 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[s// 401 is bad keyswitch res.StatusCode {case http.StatusOK:- return json.Valid(bodyBytes), nil, nil+ return json.Valid(bodyBytes), nil, analysisInfo, nilcase http.StatusForbidden:// check if the user account is blocked or notstringBody := string(bodyBytes)if strings.Contains(stringBody, BlockedUserMessage) {return true, map[string]string{"blocked": "True",- }, nil+ }, analysisInfo, nil}// Good key but not the right scope- return true, nil, nil+ return true, nil, analysisInfo, nilcase http.StatusUnauthorized:// Nothing to do; zero values are the ones we want- return false, nil, nil+ return false, nil, nil, nildefault:- return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)}}- return false, nil, nil+ return false, nil, nil, nil}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/gitlab/v2/gitlab_v2.go b/pkg/detectors/gitlab/v2/gitlab_v2.goindex 18e55084d..5bfb3f42d 100644--- a/pkg/detectors/gitlab/v2/gitlab_v2.go+++ b/pkg/detectors/gitlab/v2/gitlab_v2.go@@ -59,16 +59,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- isVerified, extraData, verificationErr := s.verifyGitlab(ctx, resMatch)+ isVerified, extraData, analysisInfo, verificationErr := s.verifyGitlab(ctx, resMatch)s1.Verified = isVerifiedfor key, value := range extraData {s1.ExtraData[key] = value}s1.SetVerificationError(verificationErr, resMatch)- s1.AnalysisInfo = map[string]string{- "key": resMatch,- }+ s1.AnalysisInfo = analysisInfo}results = append(results, s1)@@ -77,7 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, error) {+func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[string]string, map[string]string, error) {// there are 4 read 'scopes' for a gitlab token: api, read_user, read_repo, and read_registry// they all grant access to different parts of the API. I couldn't find an endpoint that every// one of these scopes has access to, so we just check an example endpoint for each scope. If any@@ -96,13 +94,18 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[sreq.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))res, err := client.Do(req)if err != nil {- return false, nil, err+ return false, nil, nil, err}defer res.Body.Close()bodyBytes, err := io.ReadAll(res.Body)if err != nil {- return false, nil, err+ return false, nil, nil, err+ }++ analysisInfo := map[string]string{+ "key": resMatch,+ "host": baseURL,}// 200 means good key and has `read_user` scope@@ -110,27 +113,27 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, map[s// 401 is bad keyswitch res.StatusCode {case http.StatusOK:- return true, nil, nil+ return true, nil, analysisInfo, nilcase http.StatusForbidden:// check if the user account is blocked or notstringBody := string(bodyBytes)if strings.Contains(stringBody, v1.BlockedUserMessage) {return true, map[string]string{"blocked": "True",- }, nil+ }, analysisInfo, nil}// Good key but not the right scope- return true, nil, nil+ return true, nil, analysisInfo, nilcase http.StatusUnauthorized:// Nothing to do; zero values are the ones we want- return false, nil, nil+ return false, nil, nil, nildefault:- return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ return false, nil, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)}}- return false, nil, nil+ return false, nil, nil, nil}func (s Scanner) Type() detectorspb.DetectorType {