Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
feat(azure): improve connstring matching (#2097)
Richard Gomezf19e2c14cdec
diff --git a/pkg/detectors/azure_storage/storage.go b/pkg/detectors/azure_storage/storage.gonew file mode 100644index 000000000..8a56044ca--- /dev/null+++ b/pkg/detectors/azure_storage/storage.go@@ -0,0 +1,197 @@+package azure_storage++import (+ "context"+ "crypto/hmac"+ "crypto/sha256"+ "encoding/base64"+ "encoding/xml"+ "fmt"+ "io"+ "net/http"+ "strings"+ "time"++ regexp "github.com/wasilibs/go-re2"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++type Scanner struct {+ client *http.Client+ detectors.MultiPartCredentialProvider+}++var _ detectors.Detector = (*Scanner)(nil)++var (+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses++ namePat = regexp.MustCompile(`(?i:Account[_.-]?Name|Storage[_.-]?(?:Account|Name))(?:.|\s){0,20}?\b([a-z0-9]{3,24})\b|([a-z0-9]{3,24})(?i:\.blob\.core\.windows\.net)`) // Names can only be lowercase alphanumeric.+ // https://learn.microsoft.com/en-us/azure/storage/common/storage-use-emulator+ testNames = map[string]struct{}{+ "devstoreaccount1": {},+ "storagesample": {},+ }++ keyPat = regexp.MustCompile(`(?i:(?:Access|Account|Storage)[_.-]?Key)(?:.|\s){0,25}?([a-zA-Z0-9+\/-]{86,88}={0,2})`)+ testKeys = map[string]struct{}{+ "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==": {},+ }+)++func (s Scanner) Keywords() []string {+ return []string{+ "DefaultEndpointsProtocol=http", "EndpointSuffix", "core.windows.net",+ "AccountName", "Account_Name", "Account.Name", "Account-Name",+ "StorageAccount", "Storage_Account", "Storage.Account", "Storage-Account",+ "AccountKey", "Account_Key", "Account.Key", "Account-Key",+ }+}++func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_AzureStorage+}++func (s Scanner) Description() string {+ return "Azure Storage is a Microsoft-managed cloud service that provides storage that is highly available, secure, durable, scalable, and redundant. Azure Storage Account keys can be used to access and manage data within storage accounts."+}++func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ // Deduplicate results.+ names := make(map[string]struct{})+ for _, matches := range namePat.FindAllStringSubmatch(dataStr, -1) {+ var name string+ if matches[1] != "" {+ name = matches[1]+ } else {+ name = matches[2]+ }+ if _, ok := testNames[name]; ok {+ continue+ }+ names[name] = struct{}{}+ }+ if len(names) == 0 {+ return results, nil+ }++ keys := make(map[string]struct{})+ for _, matches := range keyPat.FindAllStringSubmatch(dataStr, -1) {+ key := matches[1]+ if _, ok := testKeys[key]; ok {+ continue+ }+ keys[key] = struct{}{}+ }+ if len(keys) == 0 {+ return results, nil+ }++ // Check results.+ for name := range names {+ var s1 detectors.Result+ for key := range keys {+ s1 = detectors.Result{+ DetectorType: s.Type(),+ Raw: []byte(key),+ RawV2: []byte(fmt.Sprintf(`{"accountName":"%s","accountKey":"%s"}`, name, key)),+ ExtraData: map[string]string{+ "Account_name": name,+ },+ }++ if verify {+ client := s.client+ if client == nil {+ client = defaultClient+ }++ isVerified, verificationErr := s.verifyMatch(ctx, client, name, key, s1.ExtraData)+ s1.Verified = isVerified+ s1.SetVerificationError(verificationErr, key)+ }++ results = append(results, s1)+ if s1.Verified {+ break+ }+ }+ }++ return results, nil+}++type storageResponse struct {+ Containers struct {+ Container []container `xml:"Container"`+ } `xml:"Containers"`+}++type container struct {+ Name string `xml:"Name"`+}++func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, name string, key string, extraData map[string]string) (bool, error) {+ // https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key+ now := time.Now().UTC().Format(http.TimeFormat)+ stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + name + "/\ncomp:list"+ accountKeyBytes, _ := base64.StdEncoding.DecodeString(key)+ h := hmac.New(sha256.New, accountKeyBytes)+ h.Write([]byte(stringToSign))+ signature := base64.StdEncoding.EncodeToString(h.Sum(nil))++ url := "https://" + name + ".blob.core.windows.net/?comp=list"+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)+ if err != nil {+ return false, err+ }++ req.Header.Set("x-ms-date", now)+ req.Header.Set("x-ms-version", "2019-12-12")+ req.Header.Set("Authorization", "SharedKey "+name+":"+signature)++ res, err := client.Do(req)+ if err != nil {+ // If the host is not found, we can assume that the accountName is not valid+ if strings.Contains(err.Error(), "no such host") {+ return false, nil+ }+ return false, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()++ switch res.StatusCode {+ case http.StatusOK:+ // parse response+ response := storageResponse{}+ if err := xml.NewDecoder(res.Body).Decode(&response); err != nil {+ return false, err+ }++ // update the extra data with container names only+ if len(response.Containers.Container) > 0 {+ var b strings.Builder+ for i, c := range response.Containers.Container {+ if i > 0 {+ b.WriteString(", ")+ }+ b.WriteString(c.Name)+ }+ extraData["container_names"] = b.String()+ }++ return true, nil+ case http.StatusForbidden:+ // 403 if account id or key is invalid, or if the account is disabled+ return false, nil+ default:+ return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ }+}diff --git a/pkg/detectors/azurestorage/azurestorage_integration_test.go b/pkg/detectors/azure_storage/storage_integration_test.gosimilarity index 99%rename from pkg/detectors/azurestorage/azurestorage_integration_test.gorename to pkg/detectors/azure_storage/storage_integration_test.goindex 1e45f122e..85c742b84 100644--- a/pkg/detectors/azurestorage/azurestorage_integration_test.go+++ b/pkg/detectors/azure_storage/storage_integration_test.go@@ -1,7 +1,7 @@//go:build detectors// +build detectors-package azurestorage+package azure_storageimport ("context"diff --git a/pkg/detectors/azure_storage/storage_test.go b/pkg/detectors/azure_storage/storage_test.gonew file mode 100644index 000000000..cf1689837--- /dev/null+++ b/pkg/detectors/azure_storage/storage_test.go@@ -0,0 +1,328 @@+package azure_storage++import (+ "context"+ "testing"++ "github.com/google/go-cmp/cmp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"+)++func TestAzureStorage_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+ tests := []struct {+ name string+ input string+ want []string+ }{+ // True Positive+ // CONNECTION STRINGS+ {+ name: `connection_string_1`,+ input: `DefaultEndpointsProtocol=https;AccountName=storagetest123;AccountKey=YutGV0Vlauqsobd5tPWz2AKwHhBXMEWsAH+rSbz0UZUfaMVj1CFrcNQK47ygmrC4vHmc7eOp1LdM+AStk5mMyA==;EndpointSuffix=core.windows.net`,+ want: []string{`{"accountName":"storagetest123","accountKey":"YutGV0Vlauqsobd5tPWz2AKwHhBXMEWsAH+rSbz0UZUfaMVj1CFrcNQK47ygmrC4vHmc7eOp1LdM+AStk5mMyA=="}`},+ },+ {+ name: `connection_string_2`,+ input: `EndpointSuffix=core.windows.net;AccountKey=ldlKgoKPJhRjPJTkaC5c/QNqtu4sVQRc/teGJ0MZHbDYEHdvBV5z8JEfJK+evE87D7U8TzMZ0G2C+ASt2B4ifg==;AccountName=storagetest123;DefaultEndpointsProtocol=http`,+ want: []string{`{"accountName":"storagetest123","accountKey":"ldlKgoKPJhRjPJTkaC5c/QNqtu4sVQRc/teGJ0MZHbDYEHdvBV5z8JEfJK+evE87D7U8TzMZ0G2C+ASt2B4ifg=="}`},+ },+ {+ name: `connection_string_3`,+ input: ` public const string SharedStorageKey = "DefaultEndpointsProtocol=https;AccountName=huntappstorage;AccountKey=rrttFty/b52ET/e8VqpMSN+ZqAUP7hcXVkdekrPX58gsMZyOCrE+igN07t3lyi7tAV0+OrJFBaDtMe06YJ2tFw==;EndpointSuffix=core.windows.net";`,+ want: []string{`{"accountName":"huntappstorage","accountKey":"rrttFty/b52ET/e8VqpMSN+ZqAUP7hcXVkdekrPX58gsMZyOCrE+igN07t3lyi7tAV0+OrJFBaDtMe06YJ2tFw=="}`},+ },+ {+ name: `connection_string_multiline`,+ input: `+export const DevelopmentConnectionString = 'DefaultEndpointsProtocol=http;AccountName=macdemostorage;+ AccountKey=Jby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;+ QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;';`,+ want: []string{`{"accountName":"macdemostorage","accountKey":"Jby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="}`},+ },++ // LANGUAGES+ // TODO:+ // - https://github.com/Satyamk21/az204/blob/75f340c5bbfb34c1477a6885e216d5ae0972a380/Lab%203.txt#L22++ // https://github.com/facebookincubator/velox/blob/98e958c0df498efd7cf44a2078cc71caeb7aed23/velox/connectors/hive/storage_adapters/abfs/tests/AzuriteServer.h#L32-L36+ {+ name: `cpp`,+ input: `static const std::string AzuriteAccountName{"storagetest123"};+static const std::string AzuriteContainerName{"test"};+// the default key of Azurite Server used for connection+static const std::string AzuriteAccountKey{+ "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="};`,+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},+ },+ // https://github.com/MicrosoftDX/Dash/blob/03c4bb55f9e84fd03ee943559c128c4d5c2a31c2/DashServer.Tests/RequestAuthTests.cs#L29+ {+ name: `dotnet1`,+ input: ` _ctx = InitializeConfigAndCreateTestBlobs(ctx, "datax1", new Dictionary<string, string>+ {+ { "AccountName", "dashstorage1" },+ { "AccountKey", "8jqRVtXUWiEthgIhR+dFwrB8gh3lFuquvJQ1v4eabObIj7okI1cZIuzY8zZHmEdpcC0f+XlUkbFwAhjTfyrLIg==" },+ { "SecondaryAccountKey", "Klari9ZbVdFQ35aULCfqqehCsd136amhusMHWynTpz2Pg+GyQMJw3GH177hvEQbaZ2oeRYk3jw0mIaV3ehNIRg==" },+ },`,+ want: []string{+ `{"accountName":"dashstorage1","accountKey":"8jqRVtXUWiEthgIhR+dFwrB8gh3lFuquvJQ1v4eabObIj7okI1cZIuzY8zZHmEdpcC0f+XlUkbFwAhjTfyrLIg=="}`,+ `{"accountName":"dashstorage1","accountKey":"Klari9ZbVdFQ35aULCfqqehCsd136amhusMHWynTpz2Pg+GyQMJw3GH177hvEQbaZ2oeRYk3jw0mIaV3ehNIRg=="}`,+ },+ },+ // https://github.com/Satyamk21/az204/blob/75f340c5bbfb34c1477a6885e216d5ae0972a380/Lab%203.txt#L11+ {+ name: `dotnet2`,+ input: `public class Program+{+ private const string blobServiceEndpoint = "https://k21storagemedia.blob.core.windows.net/";++ private const string storageAccountName = "k21storagemedia";++ private const string storageAccountKey = "DFdukxfl0SwO4NB91bi/FTPh9BMEKr6Z5wWf+tGDfXMakXvGVp/NDzAUjWc/9171OqoDvXSj1o8N+AStUk1GXg==";+++ //The following code to create a new asynchronous Main method+ public static async Task Main(string[] args)`,+ want: []string{`{"accountName":"k21storagemedia","accountKey":"DFdukxfl0SwO4NB91bi/FTPh9BMEKr6Z5wWf+tGDfXMakXvGVp/NDzAUjWc/9171OqoDvXSj1o8N+AStUk1GXg=="}`},+ },+ // https://github.com/apache/camel/blob/main/test-infra/camel-test-infra-azure-common/src/test/java/org/apache/camel/test/infra/azure/common/services/AzuriteContainer.java#L25-L27+ {+ name: `java`,+ input: `public class AzuriteContainer extends GenericContainer<AzuriteContainer> {+ public static final String DEFAULT_ACCOUNT_NAME = "storagetest123";+ public static final String DEFAULT_ACCOUNT_KEY+ = "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==";++ public static final String IMAGE_NAME = "mcr.microsoft.com/azure-storage/azurite:3.27.0";`,+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},+ },+ // https://github.com/Azure/azure-storage-node/blob/6873387fc65bad6d577babe278be2ee2e6071493/test/common/connectionstringparsertests.js+ {+ name: `javascript`,+ input: ` var parsedConnectionString = ServiceSettings.parseAndValidateKeys(defaultConnectionString + endpointsConnectionString, validKeys);+ assert.equal(parsedConnectionString['DefaultEndpointsProtocol'], 'https');+ assert.equal(parsedConnectionString['AccountName'], 'storagetest123');+ assert.equal(parsedConnectionString['AccountKey'], 'KWPLd0rpW2T0U7K2pVpF8rYr1BgYtR7wYQk33AYiXeUoquiaY6o0TWqduxmPHlqeCNZ3LU0DHptbeIHy5l/Yhg==');+ assert.equal(parsedConnectionString['BlobEndpoint'], 'myBlobEndpoint');+ assert.equal(parsedConnectionString['QueueEndpoint'], 'myQueueEndpoint');+ assert.equal(parsedConnectionString['TableEndpoint'], 'myTableEndpoint');`,+ want: []string{`{"accountName":"storagetest123","accountKey":"KWPLd0rpW2T0U7K2pVpF8rYr1BgYtR7wYQk33AYiXeUoquiaY6o0TWqduxmPHlqeCNZ3LU0DHptbeIHy5l/Yhg=="}`},+ },+ // https://github.com/nextcloud/server/blob/81a9e19ace190ea0a64d52d95d341e25c7ad618b/tests/preseed-config.php#L89+ {+ name: `php`,+ input: ` 'arguments' => [+ 'container' => 'test',+ 'account_name' => 'storagetest123',+ 'account_key' => 'qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==',+ 'endpoint' => 'http://' . (getenv('DRONE') === 'true' ? 'azurite' : 'localhost') . ':10000/devstoreaccount1',+ 'autocreate' => true+ ]`,+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},+ },+ // https://github.com/Azure/azure-sdk-for-js/blob/2719dcfbe835a2da3003876dcb5d77efba95f912/sdk/cosmosdb/cosmos/test/public/common/_fakeTestSecrets.ts+ {+ name: `typescript`,+ input: `export const name =+ process.env.ACCOUNT_NAME || "storagename123";+export const key =+ process.env.ACCOUNT_KEY ||+ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==";`,+ want: []string{`{"accountName":"storagename123","accountKey":"C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="}`},+ },++ // FORMATS+ // TODO: Doesn't work.+ // https://github.com/Azure/azure-quickstart-templates/blob/03e792429fbc65c9353335611933746364590b22/quickstarts/microsoft.datafactory/data-factory-hive-transformation/azuredeploy.parameters.json#L9C38-L9C38+ // {+ // name: `json`,+ // input: ` "storageAccountName": {+ // "value": "changemeazurestorage"+ //},+ //"storageAccountKey": {+ // "value": "YA1gKAMY34PeVgEWPF8FdbQO+U0nFkd3SaFE4d32K16AYL/DowrTYun8anOdAiCnMkCiRYm+PxUh5mw7a7lVcA=="+ //},`,+ // want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},+ // },+ // https://github.com/ClickHouse/ClickHouse/blob/eba52b318d67d85330c9c1781499b7ff27fb7c0e/tests/integration/test_storage_azure_blob_storage/configs/named_collections.xml+ {+ name: `xml`,+ input: ` <azure_conf2>+ <account_name>storagetest123</account_name>+ <account_key>qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==</account_key>+ </azure_conf2>`,+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},+ },+ // https://github.com/hubblestack/hubble/blob/f9b7bf38752bd16b27d050a3b8787652a1c6319b/hubblestack/fileserver/azurefs.py+ {+ name: `yaml1`,+ input: ` azurefs:+ - account_name: mystorage+ account_key: 'fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA=='+ container_name: my_container+ proxy: 10.10.10.10:8080`,+ want: []string{`{"accountName":"mystorage","accountKey":"fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA=="}`},+ },+ {+ name: `yaml2`,+ input: ` - name: filesharevolume+ azureFile:+ sharename: containershare+ storageAccountName: newstore100033323+ storageAccountKey: Ar4/2iY8L0rEMeQaijINnfaMJr7vqjfbPgmJayw6Pu5l9ZI+GrFDm1uIWOqXk5RQLrTiXfBwWY6hAbPEIQqy1g==`,+ want: []string{`{"accountName":"newstore100033323","accountKey":"Ar4/2iY8L0rEMeQaijINnfaMJr7vqjfbPgmJayw6Pu5l9ZI+GrFDm1uIWOqXk5RQLrTiXfBwWY6hAbPEIQqy1g=="}`},+ },+ // This was manually base64-decoded since that doesn't work in unit tests.+ // https://github.com/fabric8io/configmapcontroller/blob/master/vendor/k8s.io/kubernetes/examples/azure_file/secret/azure-secret.yaml+ {+ name: `yaml_3`,+ input: `apiVersion: v1+ kind: Secret+ metadata:+ name: azure-secret+ type: Opaque+ data:+ azurestorageaccountname: k8stest+ azurestorageaccountkey: xIF1zJbnnojFLMSkBp50mx02rHsMK2sjU7mFt4L13hoB7drAaJ8jD6+A443jJogV7y2FUOhQCWPmM6YaNHy7qg==+`,+ want: []string{`{"accountName":"k8stest","accountKey":"xIF1zJbnnojFLMSkBp50mx02rHsMK2sjU7mFt4L13hoB7drAaJ8jD6+A443jJogV7y2FUOhQCWPmM6YaNHy7qg=="}`},+ },++ // MISC+ // https://github.com/Azure-Samples/nested-virtualization-image-builder/blob/cf0373a421343b00ce3d261be99ddced80deb55b/README.md?plain=1#L54+ {+ name: `blob_url`,+ input: `"name": "storagetest123.blob.core.windows.net", "accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="`,+ want: []string{`{"accountName":"storagetest123","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`},+ },+ {+ name: `random_cli_1`,+ input: `go run .\main.go -debug -dest="https://kenfau.blob.core.windows.net/ss3/" -AzureDefaultAccountName="kenfoo" -AzureDefaultAccountKey="hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="`,+ want: []string{+ `{"accountName":"kenfau","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`,+ `{"accountName":"kenfoo","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`,+ },+ },+ // - https://github.com/nwoolls/AzureStorageCleanup/blob/980e5cb163c78e9446e70d2513ba5a7ed9051a7a/README.md?plain=1#L24+ {+ name: `random_cli_2`,+ input: `AzureStorageCleanup.exe+ -storagename storageaccount+ -storagekey hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w==+ -container sqlbackup+ -mindaysold 60+ -searchpattern .*+ -recursive+ -whatif`,+ want: []string{`{"accountName":"storageaccount","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`},+ },+ // https://github.com/dahlej/rpi-spark-titantic/blob/d00b8f5b4696aeb2113e9452c24bb31b7f9a0242/tmp.txt#L9+ {+ name: `random_cli_3`,+ input: `$ bin/spark-submit --master \+ k8s://test-cluster.eastus2.cloudapp.azure.com:443 \+ --deploy-mode cluster \+ --name copyLocations \+ --class io.timpark.CopyData \+ --conf spark.copydata.containerpath=wasb://[email protected] \+ --conf spark.copydata.storageaccount=storagetest123 \+ --conf spark.copydata.storageaccountkey=hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w== \`,+ want: []string{`{"accountName":"storagetest123","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`},+ },+ {+ name: `custom_config_1`,+ input: `driver := ArtifactDriver{+ AccountName: "storagetest123",+ AccountKey: "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==",+ Container: "test",+ }`,+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},+ },+ // https://github.com/MicrosoftDX/Dash/blob/master/LoadTestDotNet/GetBlobCoded.cs+ {+ name: `storage_account_1`,+ input: ` this.Context.Add("StorageEndPoint", "http://dashstorage3.blob.core.windows.net");+ this.Context.Add("StorageAccount", "dashstorage3");+ this.Context.Add("AccountKey", "TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ==");+ this.Context.Add("SendChunked", false);`,+ want: []string{`{"accountName":"dashstorage3","accountKey":"TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="}`},+ },+ // https://github.com/kubecost/poc-common-configurations/blob/d626a48824a104e3089fc66ef57029f1e2212f6a/keys.txt#L18+ {+ name: `storage_account_2`,+ input: `AZ_cloud_integration_subscriptionId:0bd50fdf-c923-4e1e-850c-196ddSAMPLE+AZ_cloud_integration_azureStorageAccount:kubecostexport+AZ_cloud_integration_azureStorageAccessKey:TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ==+AZ_cloud_integration_azureStorageContainer:costexports`,+ want: []string{`{"accountName":"kubecostexport","accountKey":"TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="}`},+ },++ // False positives+ {+ name: `test_key`,+ input: ` azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;`,+ },+ {+ name: `test_key_multiline`,+ input: `+export const DevelopmentConnectionString = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;+ AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;+ QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;';`,+ },+ {+ name: `invalid_key_1`,+ input: ` docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net"`,+ },+ {+ name: `invalid_key_2`,+ input: `PS C:\> Add-AzIotHubRoutingEndpoint -ResourceGroupName "myresourcegroup" -Name "myiothub" -EndpointName S1 -EndpointType AzureStorageContainer -EndpointResourceGroup resourcegroup1 -EndpointSubscriptionId 91d12343-a3de-345d-b2ea-135792468abc -ConnectionString 'DefaultEndpointsProtocol=https;AccountName=mystorage1;AccountKey=*****;EndpointSuffix=core.windows.net' -ContainerName container -Encoding json`,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ if diff := cmp.Diff(test.want, results); diff != "" {+ t.Errorf("%s expected %d results, received %d: (-want +got)\n%s", test.name, len(test.want), len(results), diff)+ }+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}diff --git a/pkg/detectors/azurestorage/azurestorage.go b/pkg/detectors/azurestorage/azurestorage.godeleted file mode 100644index 62797e361..000000000--- a/pkg/detectors/azurestorage/azurestorage.go+++ /dev/null@@ -1,146 +0,0 @@-package azurestorage--import (- "context"- "crypto/hmac"- "crypto/sha256"- "encoding/base64"- "encoding/xml"- "fmt"- "io"- "net/http"- "strings"- "time"-- regexp "github.com/wasilibs/go-re2"-- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--type Scanner struct {- client *http.Client-}--var _ detectors.Detector = (*Scanner)(nil)--var (- defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses- keyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P<account_name>[^;]+);AccountKey=(?P<account_key>[^;]+);EndpointSuffix=core\.windows\.net`)-)--type storageResponse struct {- Containers struct {- Container []container `xml:"Container"`- } `xml:"Containers"`-}--type container struct {- Name string `xml:"Name"`-}--func (s Scanner) Keywords() []string {- return []string{"DefaultEndpointsProtocol=https;AccountName="}-}--func (s Scanner) getClient() *http.Client {- if s.client != nil {- return s.client- }- return defaultClient-}--func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- dataStr := string(data)-- matches := keyPat.FindAllStringSubmatch(dataStr, -1)-- for _, match := range matches {- if len(match) != 3 {- continue- }- accountName := strings.TrimSpace(match[1])- accountKey := strings.TrimSpace(match[2])-- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_AzureStorage,- Raw: []byte(accountKey),- ExtraData: map[string]string{- "account_name": accountName,- },- }-- if verify {- client := s.getClient()-- isVerified, verificationErr := verifyAzureStorageKey(ctx, client, accountName, accountKey, s1.ExtraData)- s1.Verified = isVerified- s1.SetVerificationError(verificationErr, accountKey)- }-- results = append(results, s1)- }-- return results, nil-}--func verifyAzureStorageKey(ctx context.Context, client *http.Client, accountName, accountKey string, extraData map[string]string) (bool, error) {- now := time.Now().UTC().Format(http.TimeFormat)- stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + accountName + "/\ncomp:list"- accountKeyBytes, _ := base64.StdEncoding.DecodeString(accountKey)- h := hmac.New(sha256.New, accountKeyBytes)- h.Write([]byte(stringToSign))- signature := base64.StdEncoding.EncodeToString(h.Sum(nil))-- url := "https://" + accountName + ".blob.core.windows.net/?comp=list"- req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)- if err != nil {- return false, err- }- req.Header.Set("x-ms-date", now)- req.Header.Set("x-ms-version", "2019-12-12")- req.Header.Set("Authorization", "SharedKey "+accountName+":"+signature)-- res, err := client.Do(req)- if err != nil {- // If the host is not found, we can assume that the accountName is not valid- if strings.Contains(err.Error(), "no such host") {- return false, nil- }- return false, err- }- defer res.Body.Close()-- switch res.StatusCode {- case http.StatusOK:- body, err := io.ReadAll(res.Body)- // ignore the error if failed because our keys has been verified.- if err == nil {- // parse response- response := storageResponse{}- if err := xml.Unmarshal(body, &response); err == nil {- // update the extra data with container names only- var containerNames []string- for _, c := range response.Containers.Container {- containerNames = append(containerNames, c.Name)- }- extraData["container_names"] = strings.Join(containerNames, ", ")- }- }-- return true, nil- case http.StatusForbidden:- // 403 if account id or key is invalid, or if the account is disabled- return false, nil- default:- return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)- }-}--func (s Scanner) Type() detectorspb.DetectorType {- return detectorspb.DetectorType_AzureStorage-}--func (s Scanner) Description() string {- return "Azure Storage is a Microsoft-managed cloud service that provides storage that is highly available, secure, durable, scalable, and redundant. Azure Storage Account keys can be used to access and manage data within storage accounts."-}diff --git a/pkg/detectors/azurestorage/azurestorage_test.go b/pkg/detectors/azurestorage/azurestorage_test.godeleted file mode 100644index 676900289..000000000--- a/pkg/detectors/azurestorage/azurestorage_test.go+++ /dev/null@@ -1,86 +0,0 @@-package azurestorage--import (- "context"- "testing"-- "github.com/google/go-cmp/cmp"-- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"-)--var (- validPattern = `- azure storage:- DefaultEndpointsProtocol=https;AccountName=myStorageAccount;AccountKey=myAccountKey12345==;EndpointSuffix=core.windows.net- `- invalidPattern = `- azure storage:- DefaultEndpointsProtocol=https;AccountName=;AccountKey=myAccountKey12345==;EndpointSuffix=core.ubuntu.net- `-)--func TestAzureStorage_Pattern(t *testing.T) {- d := Scanner{}- ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})-- tests := []struct {- name string- input string- want []string- }{- {- name: "valid pattern",- input: validPattern,- want: []string{"myAccountKey12345=="},- },- {- name: "invalid pattern",- input: invalidPattern,- want: nil,- },- }-- for _, test := range tests {- t.Run(test.name, func(t *testing.T) {- matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))- if len(matchedDetectors) == 0 {- t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)- return- }-- results, err := d.FromData(context.Background(), false, []byte(test.input))- if err != nil {- t.Errorf("error = %v", err)- return- }-- if len(results) != len(test.want) {- if len(results) == 0 {- t.Errorf("did not receive result")- } else {- t.Errorf("expected %d results, only received %d", len(test.want), len(results))- }- return- }-- actual := make(map[string]struct{}, len(results))- for _, r := range results {- if len(r.RawV2) > 0 {- actual[string(r.RawV2)] = struct{}{}- } else {- actual[string(r.Raw)] = struct{}{}- }- }- expected := make(map[string]struct{}, len(test.want))- for _, v := range test.want {- expected[v] = struct{}{}- }-- if diff := cmp.Diff(expected, actual); diff != "" {- t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)- }- })- }-}diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.goindex 888fbb2d9..ff4a83b13 100644--- a/pkg/engine/defaults/defaults.go+++ b/pkg/engine/defaults/defaults.go@@ -73,11 +73,11 @@ import (azure_serviceprincipal_v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v1"azure_serviceprincipal_v2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v2""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_openai"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_storage""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchadminkey""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchquerykey"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurestorage""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baremetrics""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beamer"@@ -899,7 +899,7 @@ func buildDetectorList() []detectors.Detector {&azure_openai.Scanner{},&azuresearchadminkey.Scanner{},&azuresearchquerykey.Scanner{},- &azurestorage.Scanner{},+ &azure_storage.Scanner{},&bannerbear.Scanner{},&baremetrics.Scanner{},&beamer.Scanner{},