Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Endpoint customizer refresh (#3308)
Kashif Khan4b6957df66e7
diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.goindex 98841f984..c281cb4a4 100644--- a/pkg/detectors/artifactory/artifactory.go+++ b/pkg/detectors/artifactory/artifactory.go@@ -15,11 +15,13 @@ import (type Scanner struct {client *http.Clientdetectors.DefaultMultiPartCredentialProvider+ detectors.EndpointSetter}var (// Ensure the Scanner satisfies the interface at compile time.- _ detectors.Detector = (*Scanner)(nil)+ _ detectors.Detector = (*Scanner)(nil)+ _ detectors.EndpointCustomizer = (*Scanner)(nil)defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses@@ -52,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif len(URLmatch) != 2 {continue}+resURLMatch = strings.TrimSpace(URLmatch[1])}@@ -61,20 +64,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}resMatch := strings.TrimSpace(match[1])- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,- Raw: []byte(resMatch),- RawV2: []byte(resMatch + resURLMatch),- }+ client := s.getClient()++ for _, URL := range s.Endpoints(resURLMatch) {+ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,+ Raw: []byte(resMatch),+ RawV2: []byte(resMatch + URL),+ }++ if verify {+ isVerified, verificationErr := verifyArtifactory(ctx, client, URL, resMatch)+ s1.Verified = isVerified+ s1.SetVerificationError(verificationErr, resMatch)+ }- if verify {- client := s.getClient()- isVerified, verificationErr := verifyArtifactory(ctx, client, resURLMatch, resMatch)- s1.Verified = isVerified- s1.SetVerificationError(verificationErr, resMatch)+ results = append(results, s1)}- results = append(results, s1)}return results, nildiff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.goindex e4743701c..b0a1c07d3 100644--- a/pkg/detectors/datadogtoken/datadogtoken.go+++ b/pkg/detectors/datadogtoken/datadogtoken.go@@ -21,8 +21,9 @@ type Scanner struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)var _ detectors.EndpointCustomizer = (*Scanner)(nil)+var _ detectors.CloudProvider = (*Scanner)(nil)-func (Scanner) DefaultEndpoint() string { return "https://api.datadoghq.com" }+func (Scanner) CloudEndpoint() string { return "https://api.datadoghq.com" }var (client = common.SaneHttpClient()@@ -126,7 +127,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {+ for _, baseURL := range s.Endpoints() {req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v2/users", nil)if err != nil {continue@@ -169,8 +170,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {-- for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {+ for _, baseURL := range s.Endpoints() {req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v1/validate", nil)if err != nil {continuediff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex b4f643f97..d3cf2f980 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -72,8 +72,14 @@ type MultiPartCredentialProvider interface {// EndpointCustomizer is an optional interface that a detector can implement to// support verifying against user-supplied endpoints.type EndpointCustomizer interface {- SetEndpoints(...string) error- DefaultEndpoint() string+ SetConfiguredEndpoints(...string) error+ SetCloudEndpoint(string)+ UseCloudEndpoint(bool)+ UseFoundEndpoints(bool)+}++type CloudProvider interface {+ CloudEndpoint() string}type Result struct {diff --git a/pkg/detectors/endpoint_customizer.go b/pkg/detectors/endpoint_customizer.goindex 85510cd9e..7cc917d85 100644--- a/pkg/detectors/endpoint_customizer.go+++ b/pkg/detectors/endpoint_customizer.go@@ -10,27 +10,43 @@ import (// of the EndpointCustomizer interface. A detector can embed this struct to// gain the functionality.type EndpointSetter struct {- endpoints []string+ configuredEndpoints []string+ cloudEndpoint string+ useCloudEndpoint bool+ useFoundEndpoints bool}-func (e *EndpointSetter) SetEndpoints(endpoints ...string) error {- if len(endpoints) == 0 {+func (e *EndpointSetter) SetConfiguredEndpoints(userConfiguredEndpoints ...string) error {+ if len(userConfiguredEndpoints) == 0 {return fmt.Errorf("at least one endpoint required")}- deduped := make([]string, 0, len(endpoints))- for _, endpoint := range endpoints {+ deduped := make([]string, 0, len(userConfiguredEndpoints))+ for _, endpoint := range userConfiguredEndpoints {common.AddStringSliceItem(endpoint, &deduped)}- e.endpoints = deduped+ e.configuredEndpoints = dedupedreturn nil}-func (e *EndpointSetter) Endpoints(defaultEndpoint string) []string {- // The only valid time len(e.endpoints) == 0 is when EndpointSetter is- // initializetd to its default state. That means SetEndpoints was never- // called and we should use the default.- if len(e.endpoints) == 0 {- return []string{defaultEndpoint}+func (e *EndpointSetter) SetCloudEndpoint(url string) {+ e.cloudEndpoint = url+}++func (e *EndpointSetter) UseCloudEndpoint(enabled bool) {+ e.useCloudEndpoint = enabled+}++func (e *EndpointSetter) UseFoundEndpoints(enabled bool) {+ e.useFoundEndpoints = enabled+}++func (e *EndpointSetter) Endpoints(foundEndpoints ...string) []string {+ endpoints := e.configuredEndpoints+ if e.useCloudEndpoint && e.cloudEndpoint != "" {+ endpoints = append(endpoints, e.cloudEndpoint)+ }+ if e.useFoundEndpoints {+ endpoints = append(endpoints, foundEndpoints...)}- return e.endpoints+ return endpoints}diff --git a/pkg/detectors/endpoint_customizer_test.go b/pkg/detectors/endpoint_customizer_test.goindex dc8542293..229cf915a 100644--- a/pkg/detectors/endpoint_customizer_test.go+++ b/pkg/detectors/endpoint_customizer_test.go@@ -10,7 +10,7 @@ func TestEmbeddedEndpointSetter(t *testing.T) {type Scanner struct{ EndpointSetter }var s Scannerassert.Equal(t, []string{"baz"}, s.Endpoints("baz"))- assert.NoError(t, s.SetEndpoints("foo", "bar"))- assert.Error(t, s.SetEndpoints())+ assert.NoError(t, s.SetConfiguredEndpoints("foo", "bar"))+ assert.Error(t, s.SetConfiguredEndpoints())assert.Equal(t, []string{"foo", "bar"}, s.Endpoints("baz"))}diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.goindex 0a78a8671..5f706e5c2 100644--- a/pkg/detectors/github/v1/github_old.go+++ b/pkg/detectors/github/v1/github_old.go@@ -19,9 +19,10 @@ type Scanner struct{ detectors.EndpointSetter }var _ detectors.Detector = (*Scanner)(nil)var _ detectors.Versioner = (*Scanner)(nil)var _ detectors.EndpointCustomizer = (*Scanner)(nil)+var _ detectors.CloudProvider = (*Scanner)(nil)-func (Scanner) Version() int { return 1 }-func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }+func (Scanner) Version() int { return 1 }+func (Scanner) CloudEndpoint() string { return "https://api.github.com" }var (// Oauth token@@ -112,7 +113,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultfunc (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token string) (bool, *UserRes, *HeaderInfo, error) {// https://developer.github.com/v3/users/#get-the-authenticated-uservar requestErr error- for _, url := range s.Endpoints(s.DefaultEndpoint()) {+ for _, url := range s.Endpoints() {requestErr = nilreq, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.goindex 08ec56758..80dede25a 100644--- a/pkg/detectors/github/v2/github.go+++ b/pkg/detectors/github/v2/github.go@@ -21,11 +21,12 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var _ detectors.Versioner = (*Scanner)(nil)var _ detectors.EndpointCustomizer = (*Scanner)(nil)+var _ detectors.CloudProvider = (*Scanner)(nil)func (s Scanner) Version() int {return 2}-func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }+func (Scanner) CloudEndpoint() string { return "https://api.github.com" }var (// Oauth tokendiff --git a/pkg/detectors/gitlab/v1/gitlab.go b/pkg/detectors/gitlab/v1/gitlab.goindex c71836622..062863822 100644--- a/pkg/detectors/gitlab/v1/gitlab.go+++ b/pkg/detectors/gitlab/v1/gitlab.go@@ -25,10 +25,11 @@ var (_ detectors.Detector = (*Scanner)(nil)_ detectors.EndpointCustomizer = (*Scanner)(nil)_ detectors.Versioner = (*Scanner)(nil)+ _ detectors.CloudProvider = (*Scanner)(nil))-func (Scanner) Version() int { return 1 }-func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" }+func (Scanner) Version() int { return 1 }+func (Scanner) CloudEndpoint() string { return "https://gitlab.com" }var (defaultClient = common.SaneHttpClient()@@ -87,7 +88,7 @@ func (s Scanner) verifyGitlab(ctx context.Context, resMatch string) (bool, errorif client == nil {client = defaultClient}- for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) {+ for _, baseURL := range s.Endpoints() {// test `read_user` scopereq, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)if err != nil {diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex d2ca05a88..dc7c61362 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -280,11 +280,19 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {}if !cfg.CustomVerifiersOnly || len(urls) == 0 {- urls = append(urls, customizer.DefaultEndpoint())+ customizer.UseFoundEndpoints(true)+ customizer.UseCloudEndpoint(true)}- if err := customizer.SetEndpoints(urls...); err != nil {++ if err := customizer.SetConfiguredEndpoints(urls...); err != nil {return false}++ cloudProvider, ok := d.(detectors.CloudProvider)+ if ok {+ customizer.SetCloudEndpoint(cloudProvider.CloudEndpoint())+ }+return true})}