Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add SonarQube Server token detector (#5419)
Kashif Khanfeea7f94e099
diff --git a/main.go b/main.goindex 0b3d72002..cdbf5230c 100644--- a/main.go+++ b/main.go@@ -595,6 +595,7 @@ func run(state overseer.State, logSync func() error) {feature.WeightsAndBiasesV2DetectorEnabled.Store(true)feature.HumioAPITokenDetectorEnabled.Store(true)feature.KongKonnectDetectorEnabled.Store(true)+ feature.SonarQubeDetectorEnabled.Store(true)conf := &config.Config{}if *configFilename != "" {diff --git a/pkg/config/detectors.go b/pkg/config/detectors.goindex 628d94af5..d76e7677c 100644--- a/pkg/config/detectors.go+++ b/pkg/config/detectors.go@@ -2,6 +2,7 @@ package configimport ("fmt"+ "net""net/url""sort""strconv"@@ -107,7 +108,7 @@ func ParseVerifierEndpoints(verifierURLs map[string]string) (map[DetectorID][]stverifierURLs[i] = rawEndpointif endpoint, err := url.Parse(rawEndpoint); err != nil {return nil, fmt.Errorf("invalid verifier url %q: %w", rawEndpoint, err)- } else if endpoint.Scheme != "https" {+ } else if endpoint.Scheme != "https" && !isLoopbackHTTP(endpoint) {return nil, fmt.Errorf("verifier url must be https: %q", rawEndpoint)}}@@ -116,6 +117,21 @@ func ParseVerifierEndpoints(verifierURLs map[string]string) (map[DetectorID][]streturn verifiers, nil}+// isLoopbackHTTP reports whether the url is plain http to this machine.+// That is safe to allow, the secret never leaves the machine. It lets+// people verify against a local server, for example http://localhost:9000.+func isLoopbackHTTP(endpoint *url.URL) bool {+ if endpoint.Scheme != "http" {+ return false+ }+ host := endpoint.Hostname()+ if host == "localhost" {+ return true+ }+ ip := net.ParseIP(host)+ return ip != nil && ip.IsLoopback()+}+func (id DetectorID) String() string {name := detector_typepb.DetectorType_name[int32(id.ID)]if name == "" {diff --git a/pkg/config/detectors_test.go b/pkg/config/detectors_test.goindex 66d74f53e..fb6cc6669 100644--- a/pkg/config/detectors_test.go+++ b/pkg/config/detectors_test.go@@ -45,3 +45,27 @@ func TestDetectorParsing(t *testing.T) {})}}++func TestParseVerifierEndpointsScheme(t *testing.T) {+ tests := []struct {+ url string+ wantErr bool+ }{+ {url: "https://sonar.example.com"},+ {url: "http://localhost:9000"},+ {url: "http://127.0.0.1:9000"},+ {url: "http://[::1]:9000"},+ {url: "http://sonar.example.com", wantErr: true},+ {url: "http://localhost.evil.com", wantErr: true},+ {url: "http://10.0.0.5:9000", wantErr: true},+ {url: "ftp://localhost", wantErr: true},+ }+ for _, test := range tests {+ t.Run(test.url, func(t *testing.T) {+ _, err := ParseVerifierEndpoints(map[string]string{"SonarQube": test.url})+ if (err != nil) != test.wantErr {+ t.Errorf("url %q: got err %v, wantErr %v", test.url, err, test.wantErr)+ }+ })+ }+}diff --git a/pkg/detectors/sonarqube/sonarqube.go b/pkg/detectors/sonarqube/sonarqube.gonew file mode 100644index 000000000..573e679bb--- /dev/null+++ b/pkg/detectors/sonarqube/sonarqube.go@@ -0,0 +1,185 @@+package sonarqube++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "slices"+ "strings"++ regexp "github.com/wasilibs/go-re2"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/ssrf"+)++// Scanner finds SonarQube Server tokens: user (squ_), global analysis (sqa_)+// and project analysis (sqp_). A token has no host in it, so it is verified+// against endpoints the user configured and hosts found in the scanned data+// (see findEndpoints).+type Scanner struct {+ detectors.DefaultMultiPartCredentialProvider+ detectors.EndpointSetter+ client *http.Client+}++// Ensure the Scanner satisfies the interface at compile time.+var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.EndpointCustomizer = (*Scanner)(nil)++// There is no public SonarQube Server, so there is no cloud endpoint.+func (Scanner) CloudEndpoint() string { return "" }++var (+ defaultClient = common.SaneHttpClient()+ keyPat = regexp.MustCompile(`\b(sq[apu]_[0-9a-f]{40})\b`)++ // Host after a sonar config key, any port, for example+ // sonar.host.url=https://sonar.example.com or SONAR_HOST_URL: sonar.example.com:8080+ hostURLPat = regexp.MustCompile(`(?i)\bsonar[._](?:host[._]url|host|url)\b["'\s:=,>]*((?:https?://)?[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?(?::\d{1,5})?)`)+)++// Keywords are used for efficiently pre-filtering chunks.+func (s Scanner) Keywords() []string {+ return []string{"squ_", "sqa_", "sqp_"}+}++// findEndpoints returns the hosts in the data named by a sonar config key,+// each one once.+func findEndpoints(data string) []string {+ var endpoints []string+ seen := make(map[string]struct{})+ for _, match := range hostURLPat.FindAllStringSubmatch(data, -1) {+ if _, ok := seen[match[1]]; !ok {+ seen[match[1]] = struct{}{}+ endpoints = append(endpoints, match[1])+ }+ }+ return endpoints+}++// FromData will find and optionally verify SonarQube secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ uniqueMatches := make(map[string]struct{})+ for _, match := range keyPat.FindAllStringSubmatch(string(data), -1) {+ uniqueMatches[match[1]] = struct{}{}+ }++ configured := s.Endpoints()+ endpoints := s.Endpoints(findEndpoints(string(data))...)++ for match := range uniqueMatches {+ result := detectors.Result{+ DetectorType: detector_typepb.DetectorType_SonarQube,+ Raw: []byte(match),+ SecretParts: map[string]string{"key": match},+ }++ // nowhere to verify, tell the user how to fix that+ if len(endpoints) == 0 {+ result.ExtraData = map[string]string{+ "message": "No SonarQube Server URL was found or configured. To verify this token, set the server URL as a custom endpoint, e.g. --verifier SonarQube=https://sonar.example.com",+ }+ }++ if verify {+ // try each endpoint, stop at the first one that accepts the token+ var verifyErr error+ for _, endpoint := range endpoints {+ // Hosts found in scanned data are not trusted, so they get a client+ // that refuses local and private addresses.+ client := s.client+ if client == nil {+ client = s.VerificationClient(defaultClient)+ if !slices.Contains(configured, endpoint) {+ client = detectors.DetectorHttpClientWithNoLocalAddresses+ }+ }++ isVerified, extraData, err := verifyMatch(ctx, client, endpoint, match)+ // a found host like localhost is the developer's machine, not ours, skip it+ if errors.Is(err, ssrf.ErrEgressBlocked) {+ continue+ }+ if err != nil {+ verifyErr = err+ continue+ }+ if isVerified {+ result.Verified = true+ result.ExtraData = extraData+ verifyErr = nil+ break+ }+ }+ result.SetVerificationError(verifyErr, match)+ }++ results = append(results, result)+ }++ return+}++type currentUserResponse struct {+ IsLoggedIn bool `json:"isLoggedIn"`+ Login string `json:"login"`+}++// verifyMatch asks the server who the token belongs to.+// The token goes in the basic auth username with an empty password.+// A wrong token gets a 401. We check isLoggedIn too, because a server+// that allows anonymous access answers 200 for any request.+func verifyMatch(ctx context.Context, client *http.Client, endpoint, token string) (bool, map[string]string, error) {+ // users pass "host:9000" or "http://host:9000", default to https+ if !strings.Contains(endpoint, "://") {+ endpoint = "https://" + endpoint+ }+ url := strings.TrimRight(endpoint, "/") + "/api/users/current"++ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)+ if err != nil {+ return false, nil, err+ }+ req.SetBasicAuth(token, "")+ req.Header.Set("Accept", "application/json")++ res, err := client.Do(req)+ if err != nil {+ return false, nil, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()++ switch res.StatusCode {+ case http.StatusOK:+ var user currentUserResponse+ if err := json.NewDecoder(res.Body).Decode(&user); err != nil {+ return false, nil, fmt.Errorf("failed to decode response: %w", err)+ }+ if !user.IsLoggedIn {+ return false, nil, nil+ }+ return true, map[string]string{"endpoint": endpoint, "login": user.Login}, nil+ case http.StatusUnauthorized:+ // the token is definitely not valid on this server+ return false, nil, nil+ default:+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ }+}++func (s Scanner) Type() detector_typepb.DetectorType {+ return detector_typepb.DetectorType_SonarQube+}++func (s Scanner) Description() string {+ return "SonarQube Server is a code quality and security analysis platform. Its user, global analysis and project analysis tokens authenticate people and CI pipelines to the SonarQube API."+}diff --git a/pkg/detectors/sonarqube/sonarqube_test.go b/pkg/detectors/sonarqube/sonarqube_test.gonew file mode 100644index 000000000..cc0cd2847--- /dev/null+++ b/pkg/detectors/sonarqube/sonarqube_test.go@@ -0,0 +1,142 @@+// There is no integration test for this detector. SonarQube Server tokens only+// work on a server the user runs, and there is no public cloud endpoint to+// verify against, so there is no live instance and credential we can keep in+// the test secrets.+package sonarqube++import (+ "context"+ "testing"++ "github.com/google/go-cmp/cmp"+ "github.com/stretchr/testify/require"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++func TestSonarQube_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "valid pattern user token",+ input: "token = squ_9747c1e823fe6696482cf7bb74e9551db87f2bdf",+ want: []string{"squ_9747c1e823fe6696482cf7bb74e9551db87f2bdf"},+ },+ {+ name: "valid pattern global analysis token",+ input: "SONAR_TOKEN=sqa_dc13d164f1302e6b4c1f403e58ab80c545491887",+ want: []string{"sqa_dc13d164f1302e6b4c1f403e58ab80c545491887"},+ },+ {+ name: "valid pattern project analysis token",+ input: `property "sonar.token","sqp_694069134ff43e17eb2a1ca18ed20b13397dce88"`,+ want: []string{"sqp_694069134ff43e17eb2a1ca18ed20b13397dce88"},+ },+ {+ name: "finds all matches",+ input: `+ [DEBUG] Using token=squ_9747c1e823fe6696482cf7bb74e9551db87f2bdf+ [DEBUG] Using token=sqa_dc13d164f1302e6b4c1f403e58ab80c545491887+ [DEBUG] Using token=sqp_694069134ff43e17eb2a1ca18ed20b13397dce88+ `,+ want: []string{+ "squ_9747c1e823fe6696482cf7bb74e9551db87f2bdf",+ "sqa_dc13d164f1302e6b4c1f403e58ab80c545491887",+ "sqp_694069134ff43e17eb2a1ca18ed20b13397dce88",+ },+ },+ {+ name: "invalid pattern",+ input: `+ [DEBUG] Using token=squ_foo+ [DEBUG] Using token=sqa_9747C1E823FE6696482CF7BB74E9551DB87F2BDF+ [DEBUG] Using token=sqp_9747c1e823fe6696482cf7bb74e9551db87f2bdf0+ [DEBUG] Using token=sqx_9747c1e823fe6696482cf7bb74e9551db87f2bdf+ `,+ want: []string{},+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("test %q failed: expected keywords %v to be found in the input", test.name, d.Keywords())+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ require.NoError(t, err)++ if len(results) != len(test.want) {+ t.Errorf("mismatch in result count: expected %d, got %d", len(test.want), len(results))+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ actual[string(r.Raw)] = struct{}{}+ }++ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}++func TestSonarQube_FindEndpoints(t *testing.T) {+ tests := []struct {+ name string+ input string+ want []string+ }{+ {name: "gradle property, any port", input: `property "sonar.host.url","http://localhost:9999"`, want: []string{"http://localhost:9999"}},+ {name: "maven property", input: "<sonar.host.url>https://sonar.example.com</sonar.host.url>", want: []string{"https://sonar.example.com"}},+ {name: "properties file", input: "sonar.host.url=https://sonar.example.com/", want: []string{"https://sonar.example.com"}},+ {name: "env var", input: "SONAR_HOST_URL=sonar.example.com:8080", want: []string{"sonar.example.com:8080"}},+ {name: "yaml", input: "SONAR_URL: https://sonar.example.com", want: []string{"https://sonar.example.com"}},+ {name: "host and port without a key is not used", input: "ssh sonar.example.com:9000", want: nil},+ {name: "docker port mapping is not a host", input: "ports: - 9000:9000", want: nil},+ {name: "same host twice is returned once", input: "sonar.host.url=https://a.example.com:9000\nSONAR_HOST_URL=https://a.example.com:9000", want: []string{"https://a.example.com:9000"}},+ {name: "variable is not a host", input: "sonar.host.url=${SONAR_URL}", want: nil},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ got := findEndpoints(test.input)+ if diff := cmp.Diff(test.want, got); diff != "" {+ t.Errorf("diff: (-want +got)\n%s", diff)+ }+ })+ }+}++func TestSonarQube_NoEndpointMessage(t *testing.T) {+ input := "token = squ_9747c1e823fe6696482cf7bb74e9551db87f2bdf"++ s := Scanner{}+ s.UseFoundEndpoints(true) // the engine turns this on by default+ results, err := s.FromData(context.Background(), true, []byte(input))+ require.NoError(t, err)+ require.Len(t, results, 1)+ require.False(t, results[0].Verified)+ require.Contains(t, results[0].ExtraData["message"], "--verifier SonarQube=")++ // with a host in the data there is something to verify, so no message+ input += "\nsonar.host.url=http://localhost:9999"+ results, err = s.FromData(context.Background(), false, []byte(input))+ require.NoError(t, err)+ require.Len(t, results, 1)+ require.Empty(t, results[0].ExtraData["message"])+}diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.goindex b54dfaffe..f739489ab 100644--- a/pkg/engine/defaults/defaults.go+++ b/pkg/engine/defaults/defaults.go@@ -734,6 +734,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/solarwindsobservability"sonarcloudv1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sonarcloud/v1"sonarcloudv2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sonarcloud/v2"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sonarqube""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/spectralops"@@ -1660,6 +1661,7 @@ func buildDetectorList() []detectors.Detector {&solarwindsobservability.Scanner{},&sonarcloudv1.Scanner{},&sonarcloudv2.Scanner{},+ &sonarqube.Scanner{},&sourcegraph.Scanner{},&sourcegraphcody.Scanner{},// &sparkpost.Scanner{},@@ -1908,6 +1910,8 @@ func buildDetectorList() []detectors.Detector {return !feature.WeightsAndBiasesV2DetectorEnabled.Load()case *kong.Scanner:return !feature.KongKonnectDetectorEnabled.Load()+ case *sonarqube.Scanner:+ return !feature.SonarQubeDetectorEnabled.Load()default:return false}diff --git a/pkg/engine/defaults/defaults_test.go b/pkg/engine/defaults/defaults_test.goindex c0848e8d6..9e567136f 100644--- a/pkg/engine/defaults/defaults_test.go+++ b/pkg/engine/defaults/defaults_test.go@@ -154,6 +154,7 @@ var excludedFromDefaultList = map[detector_typepb.DetectorType]struct{}{detector_typepb.DetectorType_HumioAPIToken: {},detector_typepb.DetectorType_Resend: {},detector_typepb.DetectorType_KongKonnect: {},+ detector_typepb.DetectorType_SonarQube: {},// Reserved / special types.detector_typepb.DetectorType_CustomRegex: {}, // added dynamically via engine config, not via buildDetectorList()diff --git a/pkg/feature/feature.go b/pkg/feature/feature.goindex 0fa1b81f9..972090299 100644--- a/pkg/feature/feature.go+++ b/pkg/feature/feature.go@@ -56,6 +56,7 @@ var (HumioAPITokenDetectorEnabled atomic.BoolEnableS3UnitProgress atomic.Bool // count each S3 bucket during a unit scan to report progress inside itKongKonnectDetectorEnabled atomic.Bool+ SonarQubeDetectorEnabled atomic.Bool)type AtomicString struct {diff --git a/pkg/pb/detector_typepb/detector_type.pb.go b/pkg/pb/detector_typepb/detector_type.pb.goindex 44c459581..38089724a 100644--- a/pkg/pb/detector_typepb/detector_type.pb.go+++ b/pkg/pb/detector_typepb/detector_type.pb.go@@ -1128,6 +1128,7 @@ const (DetectorType_HumioAPIToken DetectorType = 1070DetectorType_Resend DetectorType = 1071DetectorType_KongKonnect DetectorType = 1072+ DetectorType_SonarQube DetectorType = 1073)// Enum value maps for DetectorType.@@ -2202,6 +2203,7 @@ var (1070: "HumioAPIToken",1071: "Resend",1072: "KongKonnect",+ 1073: "SonarQube",}DetectorType_value = map[string]int32{"Alibaba": 0,@@ -3273,6 +3275,7 @@ var ("HumioAPIToken": 1070,"Resend": 1071,"KongKonnect": 1072,+ "SonarQube": 1073,})@@ -3308,7 +3311,7 @@ var File_detector_type_proto protoreflect.FileDescriptorvar file_detector_type_proto_rawDesc = []byte{0x0a, 0x13, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x2e,0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f,- 0x74, 0x79, 0x70, 0x65, 0x2a, 0xa6, 0x8c, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,+ 0x74, 0x79, 0x70, 0x65, 0x2a, 0xb6, 0x8c, 0x01, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74,0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62,0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a,0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10,@@ -4430,7 +4433,8 @@ var file_detector_type_proto_rawDesc = []byte{0x76, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x10, 0xad, 0x08, 0x12, 0x12, 0x0a, 0x0d, 0x48,0x75, 0x6d, 0x69, 0x6f, 0x41, 0x50, 0x49, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x10, 0xae, 0x08, 0x12,0x0b, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x65, 0x6e, 0x64, 0x10, 0xaf, 0x08, 0x12, 0x10, 0x0a, 0x0b,- 0x4b, 0x6f, 0x6e, 0x67, 0x4b, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x10, 0xb0, 0x08, 0x42, 0x41,+ 0x4b, 0x6f, 0x6e, 0x67, 0x4b, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x10, 0xb0, 0x08, 0x12, 0x0e,+ 0x0a, 0x09, 0x53, 0x6f, 0x6e, 0x61, 0x72, 0x51, 0x75, 0x62, 0x65, 0x10, 0xb1, 0x08, 0x42, 0x41,0x5a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75,0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75,0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70,diff --git a/proto/detector_type.proto b/proto/detector_type.protoindex af6d12da0..a3fd2a8bc 100644--- a/proto/detector_type.proto+++ b/proto/detector_type.proto@@ -1074,4 +1074,5 @@ enum DetectorType {HumioAPIToken = 1070;Resend = 1071;KongKonnect = 1072;+ SonarQube = 1073;}