Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Updated Fastly Personal Token Detector (#3386)
Kashif Khanbc32592066bb
diff --git a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.goindex 11aaacd2a..9b2866195 100644--- a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go+++ b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go@@ -4,10 +4,10 @@ import ("context""encoding/json""fmt"- regexp "github.com/wasilibs/go-re2""io""net/http"- "strings"++ regexp "github.com/wasilibs/go-re2""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"@@ -31,60 +31,34 @@ func (s Scanner) Keywords() []string {return []string{"fastly"}}-type fastlyUserRes struct {- Login string `json:"login"`- Name string `json:"name"`- Role string `json:"role"`- TwoFactorAuthEnabled bool `json:"two_factor_auth_enabled"`- Locked bool `json:"locked"`+type token struct {+ TokenID string `json:"id"`+ UserID string `json:"user_id"`+ ExpiresAt string `json:"expires_at"`+ Scope string `json:"scope"`}// FromData will find and optionally verify FastlyPersonalToken secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ var uniqueMatches = make(map[string]struct{})- for _, match := range matches {- if len(match) != 2 {- continue- }- resMatch := strings.TrimSpace(match[1])+ for _, matches := range keyPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueMatches[matches[1]] = struct{}{}+ }+ for match := range uniqueMatches {s1 := detectors.Result{DetectorType: detectorspb.DetectorType_FastlyPersonalToken,- Raw: []byte(resMatch),+ Raw: []byte(match),}if verify {- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.fastly.com/current_user", nil)- if err != nil {- continue- }- req.Header.Add("Fastly-Key", resMatch)- res, err := client.Do(req)- if err == nil {- bodyBytes, err := io.ReadAll(res.Body)- if err != nil {- continue- }- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- var userRes fastlyUserRes- err = json.Unmarshal(bodyBytes, &userRes)- if err != nil {- continue- }- s1.Verified = true- s1.ExtraData = map[string]string{- "username": userRes.Login,- "name": userRes.Name,- "role": userRes.Role,- "locked": fmt.Sprintf("%t", userRes.Locked),- "two_factor_auth_enabled": fmt.Sprintf("%t", userRes.TwoFactorAuthEnabled),- }- }- }+ extraData, verified, verificationErr := verifyFastlyApiToken(ctx, match)+ s1.Verified = verified+ s1.ExtraData = extraData+ s1.SetVerificationError(verificationErr, match)}results = append(results, s1)@@ -100,3 +74,55 @@ func (s Scanner) Type() detectorspb.DetectorType {func (s Scanner) Description() string {return "Fastly is a content delivery network (CDN) and cloud service provider. Fastly personal tokens can be used to authenticate API requests to Fastly services."}++func verifyFastlyApiToken(ctx context.Context, apiToken string) (map[string]string, bool, error) {+ // api-docs: https://www.fastly.com/documentation/reference/api/auth-tokens/user/+ req, err := http.NewRequestWithContext(ctx, "GET", "https://api.fastly.com/tokens/self", nil)+ if err != nil {+ return nil, false, err+ }++ // add api key in the header+ req.Header.Add("Fastly-Key", apiToken)+ resp, err := client.Do(req)+ if err != nil {+ return nil, false, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, resp.Body)+ _ = resp.Body.Close()+ }()++ switch resp.StatusCode {+ case http.StatusOK:+ var self token+ if err = json.NewDecoder(resp.Body).Decode(&self); err != nil {+ return nil, false, err+ }++ // capture token details in the map+ extraData := map[string]string{+ // token id is the alphanumeric string uniquely identifying a token+ "token_id": self.TokenID,+ // user id is the alphanumeric string uniquely identifying the user+ "user_id": self.UserID,+ // expires at is time-stamp (UTC) of when the token will expire+ "token_expires_at": self.ExpiresAt,+ // token scope is space-delimited list of authorization scope of the token+ "token_scope": self.Scope,+ }++ // if expires at is empty which mean token is set to never expire, add 'Never' as the value+ if extraData["token_expires_at"] == "" {+ extraData["token_expires_at"] = "never"+ }++ return extraData, true, nil+ case http.StatusUnauthorized, http.StatusForbidden:+ // as per fastly documentation: An HTTP 401 response is returned on an expired token. An HTTP 403 response is returned on an invalid access token.+ return nil, false, nil+ default:+ return nil, false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)++ }+}diff --git a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_integration_test.go b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_integration_test.gonew file mode 100644index 000000000..20eb25f7e--- /dev/null+++ b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_integration_test.go@@ -0,0 +1,126 @@+//go:build detectors+// +build detectors++package fastlypersonaltoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestFastlyPersonalToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ secret := testSecrets.MustGetField("FASTLYPERSONALTOKEN_TOKEN")+ inactiveSecret := testSecrets.MustGetField("FASTLYPERSONALTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: ctx,+ data: []byte(fmt.Sprintf("You can find a fastlypersonaltoken secret %s within", secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_FastlyPersonalToken,+ Verified: true,+ ExtraData: map[string]string{+ "token_id": "2GUTBVFzHG2zVOMGtEpi9q",+ "user_id": "2j1UhHmRhefRMNNrlxcyf5",+ "token_expires_at": "never",+ "token_scope": "global:read",+ },+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a fastlypersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_FastlyPersonalToken,+ Verified: false,+ ExtraData: map[string]string{},+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("FastlyPersonalToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("FastlyPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.goindex cbf2c5426..0d14ddfec 100644--- a/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go+++ b/pkg/detectors/fastlypersonaltoken/fastlypersonaltoken_test.go@@ -1,120 +1,94 @@-//go:build detectors-// +build detectors-package fastlypersonaltokenimport ("context"- "fmt""testing"- "time"- "github.com/kylelemons/godebug/pretty"+ "github.com/google/go-cmp/cmp"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+var (+ // example picked from: https://github.com/ryan-miller/learn-go-with-tests/blob/181467c9e512f7e68d3f3cbcea89f0050982416c/fastly/users.go#L22+ validPattern = `+ // headers and header values+ const fastlyKeyToken string = "Fastly-Key"+ const fastlyKey string = "TVAWji0p7uDI6OP9DyWvmV-vgoUoXIuf"+ const contentTypeToken string = "Content-Type"+ const appJsonContentType = "application/json"`++ validPatternToken = "TVAWji0p7uDI6OP9DyWvmV-vgoUoXIuf"++ invalidPattern = `+ // headers and header values+ const fastlyKeyToken string = "Fastly-Key"+ const fastlyKey string = "$FASTLY_KEY"+ const contentTypeToken string = "Content-Type"+ const appJsonContentType = "application/json"`)-func TestFastlyPersonalToken_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("FASTLYPERSONALTOKEN_TOKEN")- inactiveSecret := testSecrets.MustGetField("FASTLYPERSONALTOKEN_INACTIVE")+func TestFastlyPersonalToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})- type args struct {- ctx context.Context- data []byte- verify bool- }tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool+ name string+ input string+ want []string}{{- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a fastlypersonaltoken secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_FastlyPersonalToken,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a fastlypersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_FastlyPersonalToken,- Verified: false,- },- },- wantErr: false,+ name: "valid pattern",+ input: validPattern,+ want: []string{validPatternToken},},{- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,+ name: "invalid pattern",+ input: invalidPattern,+ want: nil,},}- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("FastlyPersonalToken.FromData() error = %v, wantErr %v", err, tt.wantErr)++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)return}- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return}- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("FastlyPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return}- })- }-}-func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- b.ResetTimer()- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)+ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}}}+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }})}}