Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
add SecretParts to custom detectors (#5470)
Casey Tran2caa45cf2210
diff --git a/pkg/custom_detectors/CUSTOM_DETECTORS.md b/pkg/custom_detectors/CUSTOM_DETECTORS.mdindex 95fb2165d..fa2cf1915 100644--- a/pkg/custom_detectors/CUSTOM_DETECTORS.md+++ b/pkg/custom_detectors/CUSTOM_DETECTORS.md@@ -34,6 +34,7 @@ This guide will walk you through setting up a custom detector in TruffleHog to i- **`name`**: A unique identifier for your custom detector.- **`keywords`**: An array of strings that, when found, trigger the regex search. If multiple keywords are specified, the presence of any one of them will initiate the regex search.- **`regex`**: Defines the patterns to identify potential secrets. You can specify one or more named regular expressions. For a detection to be successful, each named regex must find a match. Capture groups `()` within these regular expressions are used to extract specific portions of the matched text, enabling the detector to process and report on particular segments of the identified patterns.+ Each named regex is recorded as a separate part of the detected secret, labeled with the regex's name, with the first capture group as its value (or the entire match if there is no capture group). Renaming a regex changes that label for new results, so keep names stable once a detector is in use.- **`verify`**: An optional section to validate detected secrets. If you want to verify or unverify detected secrets, this section needs to be configured. If not configured, all detected secrets will be marked as unverified. Read [verification server examples](#verification-server-examples)- **`successRanges`**: A list of HTTP status codes (or ranges) that indicate the secret is **live** (active). When the verification server responds with a matching status code, the secret is marked as verified. Each entry can be a single code (`"200"`) or an inclusive range (`"200-202"`). If omitted (along with `rotatedRanges`), only `200` is treated as verified (backward compatible).diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.goindex 934c3f368..27f3b67ad 100644--- a/pkg/custom_detectors/custom_detectors.go+++ b/pkg/custom_detectors/custom_detectors.go@@ -282,6 +282,8 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[stringDetectorType: detector_typepb.DetectorType_CustomRegex,DetectorName: c.GetName(),ExtraData: map[string]string{},+ // Each entry is named after a regex in the user's config.+ SecretParts: make(map[string]string, len(match)),}var raw string@@ -296,6 +298,7 @@ func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string}raw += secretsecrets = append(secrets, secret)+ result.SecretParts[key] = secret// We set the full regex match as the primary secret value.// Reasoning:diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.goindex 8f56045fd..ca5569ab5 100644--- a/pkg/custom_detectors/custom_detectors_test.go+++ b/pkg/custom_detectors/custom_detectors_test.go@@ -241,6 +241,7 @@ func TestDetector(t *testing.T) {assert.NoError(t, err)assert.Equal(t, 1, len(results))assert.Equal(t, results[0].Raw, []byte(`123456`))+ assert.Equal(t, map[string]string{"regex": "123456"}, results[0].SecretParts)}func TestDetectorPrimarySecret(t *testing.T) {@@ -260,6 +261,10 @@ func TestDetectorPrimarySecret(t *testing.T) {assert.NoError(t, err)assert.Equal(t, 1, len(results))assert.Equal(t, "secret_YI7C90ACY1_yy", results[0].GetPrimarySecretValue())+ assert.Equal(t, map[string]string{+ "id": "id_ALPHA10100_yy",+ "secret": "secret_YI7C90ACY1_yy",+ }, results[0].SecretParts)}func TestDetectorPrimarySecretFullMatch(t *testing.T) {@@ -313,7 +318,75 @@ func TestDetectorPrimarySecretFullMatch(t *testing.T) {assert.Equal(t, tt.want, results[0].GetPrimarySecretValue())})}+}++// TestDetectorSecretParts checks that each named regex becomes one entry in SecretParts.+func TestDetectorSecretParts(t *testing.T) {+ t.Parallel()++ t.Run("capture group", func(t *testing.T) {+ t.Parallel()+ detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{+ Name: "test",+ Keywords: []string{"secret"},+ Regex: map[string]string{"secret": `secret *= *"([^"\r\n]+)"`},+ })+ require.NoError(t, err)++ results, err := detector.FromData(context.Background(), false, []byte(`secret="mysecret"`))+ require.NoError(t, err)+ require.Len(t, results, 1)+ assert.Equal(t, map[string]string{"secret": "mysecret"}, results[0].SecretParts)+ assert.Equal(t, `secret="mysecret"`, results[0].GetPrimarySecretValue())+ })++ t.Run("no capture group", func(t *testing.T) {+ t.Parallel()+ detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{+ Name: "test",+ Keywords: []string{"hog"},+ Regex: map[string]string{"token": `hog_[A-Za-z0-9]{8}`},+ })+ require.NoError(t, err)++ results, err := detector.FromData(context.Background(), false, []byte("prefix hog_abcd1234 suffix"))+ require.NoError(t, err)+ require.Len(t, results, 1)+ assert.Equal(t, map[string]string{"token": "hog_abcd1234"}, results[0].SecretParts)+ assert.Equal(t, []byte("hog_abcd1234"), results[0].Raw)+ })+ t.Run("verified multi-part", func(t *testing.T) {+ t.Parallel()+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ }))+ t.Cleanup(ts.Close)++ detector, err := NewWebhookCustomRegex(&custom_detectorspb.CustomRegex{+ Name: "test",+ Keywords: []string{"secret"},+ Regex: map[string]string{+ "id": `id_([A-Z]{4})`,+ "secret": `secret_([A-Z]{4})`,+ },+ Verify: []*custom_detectorspb.VerifierConfig{{+ Endpoint: ts.URL,+ Unsafe: true,+ }},+ })+ require.NoError(t, err)++ results, err := detector.FromData(context.Background(), true, []byte("id_ABCD secret_WXYZ"))+ require.NoError(t, err)+ require.Len(t, results, 1)+ assert.True(t, results[0].Verified)+ assert.Equal(t, map[string]string{+ "id": "ABCD",+ "secret": "WXYZ",+ }, results[0].SecretParts)+ assert.Equal(t, []byte("ABCDWXYZ"), results[0].Raw)+ })}func TestDetectorValidations(t *testing.T) {@@ -351,6 +424,7 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("MyStr0ngP@ssword!"),+ SecretParts: map[string]string{"password": "MyStr0ngP@ssword!"},},},},@@ -398,6 +472,7 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("MyStrongPassword!"),+ SecretParts: map[string]string{"password": "MyStrongPassword!"},},},},@@ -445,6 +520,7 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("MyStrongPassword!"),+ SecretParts: map[string]string{"password": "MyStrongPassword!"},},},},@@ -492,6 +568,7 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("MyStr@ngP@ssword!"),+ SecretParts: map[string]string{"password": "MyStr@ngP@ssword!"},},},},@@ -540,6 +617,7 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("MyStrongP@ssword"),+ SecretParts: map[string]string{"password": "MyStrongP@ssword"},},},},@@ -588,6 +666,7 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("mystrongp@ssword"),+ SecretParts: map[string]string{"password": "mystrongp@ssword"},},},},@@ -624,6 +703,10 @@ func TestDetectorValidations(t *testing.T) {DetectorName: "test",Verified: false,Raw: []byte("c392c9837d69b44c764cbf260b-e6184MyStrongP@ssword"),+ SecretParts: map[string]string{+ "api_key": "c392c9837d69b44c764cbf260b-e6184",+ "password": "MyStrongP@ssword",+ },},},},@@ -876,12 +959,12 @@ func TestVerificationWithConfigurableRanges(t *testing.T) {t.Parallel()tests := []struct {- name string- serverStatus int- successRanges []string- rotatedRanges []string- wantVerified bool- wantVerifyErr bool+ name string+ serverStatus int+ successRanges []string+ rotatedRanges []string+ wantVerified bool+ wantVerifyErr bool}{{name: "backward compat: no ranges, 200 -> verified",