Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[Feat] Detector implementation for Azure SAS Tokens (#3963)
Abdul Basit1848ff6e6360
diff --git a/pkg/detectors/azuresastoken/azuresastoken.go b/pkg/detectors/azuresastoken/azuresastoken.gonew file mode 100644index 000000000..d13b6b136--- /dev/null+++ b/pkg/detectors/azuresastoken/azuresastoken.go@@ -0,0 +1,157 @@+package azuresastoken++import (+ "context"+ "errors"+ "fmt"+ "io"+ "net/http"+ "strings"++ regexp "github.com/wasilibs/go-re2"++ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++type Scanner struct {+ client *http.Client+ detectors.DefaultMultiPartCredentialProvider+}++var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)++var (+ defaultClient = common.SaneHttpClient()++ // microsoft storage resource naming rules: https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage:~:text=format%3A%0AVaultName_KeyName_KeyVersion.-,Microsoft.Storage,-Expand%20table+ urlPat = regexp.MustCompile(`https://([a-zA-Z0-9][a-z0-9_-]{1,22}[a-zA-Z0-9])\.blob\.core\.windows\.net/[a-z0-9]([a-z0-9-]{1,61}[a-z0-9])?(?:/[a-zA-Z0-9._-]+)*`)++ keyPat = regexp.MustCompile(+ detectors.PrefixRegex([]string{"azure", "sas", "token", "blob", ".blob.core.windows.net"}) ++ `(sp=[racwdli]+&st=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z&se=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z(?:&sip=\d{1,3}(?:\.\d{1,3}){3}(?:-\d{1,3}(?:\.\d{1,3}){3})?)?(&spr=https)?(?:,https)?&sv=\d{4}-\d{2}-\d{2}&sr=[bcfso]&sig=[a-zA-Z0-9%]{10,})`)++ invalidStorageAccounts = simple.NewCache[struct{}]()++ noSuchHostErr = errors.New("no such host")+)++func (s Scanner) Keywords() []string {+ return []string{+ "azure",+ ".blob.core.windows.net",+ }+}++func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_AzureSasToken+}++func (s Scanner) Description() string {+ return "An Azure Shared Access Signature (SAS) token is a time-limited, permission-based URL query string that grants secure, granular access to Azure Storage resources (e.g., blobs, containers, files) without exposing account keys."+}++func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ logger := logContext.AddLogger(ctx).Logger().WithName("azuresas")++ dataStr := string(data)++ // deduplicate urlMatches+ urlMatchesUnique := make(map[string]string)+ for _, urlMatch := range urlPat.FindAllStringSubmatch(dataStr, -1) {+ urlMatchesUnique[urlMatch[0]] = urlMatch[1]+ }++ // deduplicate keyMatches+ keyMatchesUnique := make(map[string]struct{})+ for _, keyMatch := range keyPat.FindAllStringSubmatch(dataStr, -1) {+ keyMatchesUnique[keyMatch[1]] = struct{}{}+ }++ // Check results.+UrlLoop:+ for url, storageAccount := range urlMatchesUnique {+ for key := range keyMatchesUnique {+ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_AzureSasToken,+ Raw: []byte(url),+ RawV2: []byte(url + key),+ }++ if verify {+ if invalidStorageAccounts.Exists(storageAccount) {+ logger.V(3).Info("Skipping invalid storage account", "storage account", storageAccount)+ break+ }++ client := s.client+ if client == nil {+ client = defaultClient+ }++ isVerified, verificationErr := verifyMatch(ctx, client, url, key, true)+ s1.Verified = isVerified++ if verificationErr != nil {+ if errors.Is(verificationErr, noSuchHostErr) {+ invalidStorageAccounts.Set(storageAccount, struct{}{})+ continue UrlLoop+ }+ s1.SetVerificationError(verificationErr, key)+ }+ }++ results = append(results, s1)+ }+ }++ return results, nil+}++func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""+}++func verifyMatch(ctx context.Context, client *http.Client, url, key string, retryOn403 bool) (bool, error) {+ urlWithToken := url + "?" + key++ req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlWithToken, nil)+ if err != nil {+ return false, err+ }++ res, err := client.Do(req)+ if err != nil {+ if strings.Contains(err.Error(), "no such host") {+ return false, noSuchHostErr+ }+ return false, err+ }+ defer res.Body.Close()++ bodyBytes, err := io.ReadAll(res.Body)+ if err != nil {+ return false, err+ }++ switch res.StatusCode {+ case http.StatusOK:+ return true, nil+ case http.StatusForbidden:+ if retryOn403 && strings.Contains(string(bodyBytes), "Signature did not match") {+ // need to add additional query parameters for container urls+ // https://stackoverflow.com/questions/25038429/azure-shared-access-signature-signature-did-not-match+ return verifyMatch(ctx, client, url, key+"&comp=list&restype=container", false)+ }+ if strings.Contains(string(bodyBytes), "AuthorizationFailure") && strings.Contains(key, "&sip=") {+ return false, fmt.Errorf("SAS token is restricted to specific IP addresses")+ }+ return false, nil+ default:+ return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ }+}diff --git a/pkg/detectors/azuresastoken/azuresastoken_integration_test.go b/pkg/detectors/azuresastoken/azuresastoken_integration_test.gonew file mode 100644index 000000000..39883c9af--- /dev/null+++ b/pkg/detectors/azuresastoken/azuresastoken_integration_test.go@@ -0,0 +1,129 @@+//go:build detectors+// +build detectors++package azuresastoken++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/google/go-cmp/cmp"+ "github.com/google/go-cmp/cmp/cmpopts"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestAzureSasToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ url := testSecrets.MustGetField("AZURESASTOKEN_URL")+ secret := testSecrets.MustGetField("AZURESASTOKEN")+ inactiveSecret := testSecrets.MustGetField("AZURESASTOKEN_INACTIVE")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ wantVerificationErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure sas url %s and token %s within", url, secret)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSasToken,+ Verified: true,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a azure sas url %s and token %s within but not valid", url, inactiveSecret)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AzureSasToken,+ Verified: false,+ },+ },+ wantErr: false,+ wantVerificationErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ wantVerificationErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("AzureSasToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ if (got[i].VerificationError() != nil) != tt.wantVerificationErr {+ t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())+ }+ }+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "Redacted", "verificationError")+ if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {+ t.Errorf("AzureSasToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/azuresastoken/azuresastoken_test.go b/pkg/detectors/azuresastoken/azuresastoken_test.gonew file mode 100644index 000000000..62c8330a2--- /dev/null+++ b/pkg/detectors/azuresastoken/azuresastoken_test.go@@ -0,0 +1,129 @@+package azuresastoken++import (+ "context"+ "testing"++ "github.com/google/go-cmp/cmp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++func TestAzureSASToken_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})++ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "valid pattern",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity+ `,+ want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecuritysp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D"},+ },+ {+ name: "valid pattern with ip",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sip=168.1.6.50&spr=https&sv=2022-11-02&sr=c&sig=c%2BUXo%2FJwf%2FGHomqYaw6tyRykKMaAnyikkf8nS7btD3DYg%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity+ `,+ want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecuritysp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sip=168.1.6.50&spr=https&sv=2022-11-02&sr=c&sig=c%2BUXo%2FJwf%2FGHomqYaw6tyRykKMaAnyikkf8nS7btD3DYg%3D"},+ },+ {+ name: "valid pattern with ip range",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sip=168.1.6.50-168.1.6.80&spr=https&sv=2022-11-02&sr=c&sig=RiA6rO2VwFNZ73trWyY6fsasg0ViUp0k3sDxcl6aA1Rtg%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity+ `,+ want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecuritysp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sip=168.1.6.50-168.1.6.80&spr=https&sv=2022-11-02&sr=c&sig=RiA6rO2VwFNZ73trWyY6fsasg0ViUp0k3sDxcl6aA1Rtg%3D"},+ },+ {+ name: "valid pattern without https",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sv=2022-11-02&sr=c&sig=OYbYoPKW7vVGjFMBu2QDDW%2BlpoShcxawVHR91NQPosY8%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity+ `,+ want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecuritysp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sv=2022-11-02&sr=c&sig=OYbYoPKW7vVGjFMBu2QDDW%2BlpoShcxawVHR91NQPosY8%3D"},+ },+ {+ name: "valid pattern with blob url",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity/test_blob.txt+ `,+ want: []string{"https://trufflesecurity.blob.core.windows.net/trufflesecurity/test_blob.txtsp=r&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D"},+ },+ {+ name: "invalid pattern",+ input: `+ AZURE_BLOB_SAS_TOKEN=st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/12trufflesecurity+ `,+ want: nil,+ },+ {+ name: "invalid pattern with invalid permission",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=rqx&st=2025-03-04T07:24:52Z&se=2025-04-04T15:24:52Z&spr=https&sv=2022-11-02&sr=c&sig=WSdF9YeZhvrbs%2B%2B1f8ZdDBzEe7fBJ%2BenuaXQ%2BJ9WOw0%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/12trufflesecurity+ `,+ want: nil,+ },+ {+ name: "invalid pattern with invalid ip",+ input: `+ AZURE_BLOB_SAS_TOKEN=sp=rcwl&st=2025-03-10T06:58:25Z&se=2025-03-10T14:58:25Z&sip=168.1.6&spr=https&sv=2022-11-02&sr=c&sig=c%2BUXo%2FJwf%2FGHomqYaw6tyRykKMaAnyikkf8nS7btD3DYg%3D+ AZURE_BLOB_SAS_URL=https://trufflesecurity.blob.core.windows.net/trufflesecurity+ `,+ want: nil,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.goindex 7b65cf1c8..1c5b039b3 100644--- a/pkg/engine/defaults/defaults.go+++ b/pkg/engine/defaults/defaults.go@@ -74,6 +74,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_storage""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresastoken""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchadminkey""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchquerykey""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear"@@ -906,6 +907,7 @@ func buildDetectorList() []detectors.Detector {&azuredevopspersonalaccesstoken.Scanner{},// &azurefunctionkey.Scanner{}, // detector is throwing some FPs&azure_openai.Scanner{},+ &azuresastoken.Scanner{},&azuresearchadminkey.Scanner{},&azuresearchquerykey.Scanner{},&azure_storage.Scanner{},