Skip to content

Find, verify, and analyze leaked credentials

feat(azure): improve connstring matching (#2097)

Richard Gomezf19e2c14cdec
diff --git a/pkg/detectors/azure_storage/storage.go b/pkg/detectors/azure_storage/storage.go
new file mode 100644
index 000000000..8a56044ca
--- /dev/null
+++ b/pkg/detectors/azure_storage/storage.go
@@ -0,0 +1,197 @@
+package azure_storage
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/xml"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+ client *http.Client
+ detectors.MultiPartCredentialProvider
+}
+
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
+
+ namePat = regexp.MustCompile(`(?i:Account[_.-]?Name|Storage[_.-]?(?:Account|Name))(?:.|\s){0,20}?\b([a-z0-9]{3,24})\b|([a-z0-9]{3,24})(?i:\.blob\.core\.windows\.net)`) // Names can only be lowercase alphanumeric.
+ // https://learn.microsoft.com/en-us/azure/storage/common/storage-use-emulator
+ testNames = map[string]struct{}{
+ "devstoreaccount1": {},
+ "storagesample": {},
+ }
+
+ keyPat = regexp.MustCompile(`(?i:(?:Access|Account|Storage)[_.-]?Key)(?:.|\s){0,25}?([a-zA-Z0-9+\/-]{86,88}={0,2})`)
+ testKeys = map[string]struct{}{
+ "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==": {},
+ }
+)
+
+func (s Scanner) Keywords() []string {
+ return []string{
+ "DefaultEndpointsProtocol=http", "EndpointSuffix", "core.windows.net",
+ "AccountName", "Account_Name", "Account.Name", "Account-Name",
+ "StorageAccount", "Storage_Account", "Storage.Account", "Storage-Account",
+ "AccountKey", "Account_Key", "Account.Key", "Account-Key",
+ }
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_AzureStorage
+}
+
+func (s Scanner) Description() string {
+ return "Azure Storage is a Microsoft-managed cloud service that provides storage that is highly available, secure, durable, scalable, and redundant. Azure Storage Account keys can be used to access and manage data within storage accounts."
+}
+
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ // Deduplicate results.
+ names := make(map[string]struct{})
+ for _, matches := range namePat.FindAllStringSubmatch(dataStr, -1) {
+ var name string
+ if matches[1] != "" {
+ name = matches[1]
+ } else {
+ name = matches[2]
+ }
+ if _, ok := testNames[name]; ok {
+ continue
+ }
+ names[name] = struct{}{}
+ }
+ if len(names) == 0 {
+ return results, nil
+ }
+
+ keys := make(map[string]struct{})
+ for _, matches := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+ key := matches[1]
+ if _, ok := testKeys[key]; ok {
+ continue
+ }
+ keys[key] = struct{}{}
+ }
+ if len(keys) == 0 {
+ return results, nil
+ }
+
+ // Check results.
+ for name := range names {
+ var s1 detectors.Result
+ for key := range keys {
+ s1 = detectors.Result{
+ DetectorType: s.Type(),
+ Raw: []byte(key),
+ RawV2: []byte(fmt.Sprintf(`{"accountName":"%s","accountKey":"%s"}`, name, key)),
+ ExtraData: map[string]string{
+ "Account_name": name,
+ },
+ }
+
+ if verify {
+ client := s.client
+ if client == nil {
+ client = defaultClient
+ }
+
+ isVerified, verificationErr := s.verifyMatch(ctx, client, name, key, s1.ExtraData)
+ s1.Verified = isVerified
+ s1.SetVerificationError(verificationErr, key)
+ }
+
+ results = append(results, s1)
+ if s1.Verified {
+ break
+ }
+ }
+ }
+
+ return results, nil
+}
+
+type storageResponse struct {
+ Containers struct {
+ Container []container `xml:"Container"`
+ } `xml:"Containers"`
+}
+
+type container struct {
+ Name string `xml:"Name"`
+}
+
+func (s Scanner) verifyMatch(ctx context.Context, client *http.Client, name string, key string, extraData map[string]string) (bool, error) {
+ // https://learn.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key
+ now := time.Now().UTC().Format(http.TimeFormat)
+ stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + name + "/\ncomp:list"
+ accountKeyBytes, _ := base64.StdEncoding.DecodeString(key)
+ h := hmac.New(sha256.New, accountKeyBytes)
+ h.Write([]byte(stringToSign))
+ signature := base64.StdEncoding.EncodeToString(h.Sum(nil))
+
+ url := "https://" + name + ".blob.core.windows.net/?comp=list"
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return false, err
+ }
+
+ req.Header.Set("x-ms-date", now)
+ req.Header.Set("x-ms-version", "2019-12-12")
+ req.Header.Set("Authorization", "SharedKey "+name+":"+signature)
+
+ res, err := client.Do(req)
+ if err != nil {
+ // If the host is not found, we can assume that the accountName is not valid
+ if strings.Contains(err.Error(), "no such host") {
+ return false, nil
+ }
+ return false, err
+ }
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
+
+ switch res.StatusCode {
+ case http.StatusOK:
+ // parse response
+ response := storageResponse{}
+ if err := xml.NewDecoder(res.Body).Decode(&response); err != nil {
+ return false, err
+ }
+
+ // update the extra data with container names only
+ if len(response.Containers.Container) > 0 {
+ var b strings.Builder
+ for i, c := range response.Containers.Container {
+ if i > 0 {
+ b.WriteString(", ")
+ }
+ b.WriteString(c.Name)
+ }
+ extraData["container_names"] = b.String()
+ }
+
+ return true, nil
+ case http.StatusForbidden:
+ // 403 if account id or key is invalid, or if the account is disabled
+ return false, nil
+ default:
+ return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+ }
+}
diff --git a/pkg/detectors/azurestorage/azurestorage_integration_test.go b/pkg/detectors/azure_storage/storage_integration_test.go
similarity index 99%
rename from pkg/detectors/azurestorage/azurestorage_integration_test.go
rename to pkg/detectors/azure_storage/storage_integration_test.go
index 1e45f122e..85c742b84 100644
--- a/pkg/detectors/azurestorage/azurestorage_integration_test.go
+++ b/pkg/detectors/azure_storage/storage_integration_test.go
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
-package azurestorage
+package azure_storage
import (
"context"
diff --git a/pkg/detectors/azure_storage/storage_test.go b/pkg/detectors/azure_storage/storage_test.go
new file mode 100644
index 000000000..cf1689837
--- /dev/null
+++ b/pkg/detectors/azure_storage/storage_test.go
@@ -0,0 +1,328 @@
+package azure_storage
+
+import (
+ "context"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
+)
+
+func TestAzureStorage_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+ tests := []struct {
+ name string
+ input string
+ want []string
+ }{
+ // True Positive
+ // CONNECTION STRINGS
+ {
+ name: `connection_string_1`,
+ input: `DefaultEndpointsProtocol=https;AccountName=storagetest123;AccountKey=YutGV0Vlauqsobd5tPWz2AKwHhBXMEWsAH+rSbz0UZUfaMVj1CFrcNQK47ygmrC4vHmc7eOp1LdM+AStk5mMyA==;EndpointSuffix=core.windows.net`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"YutGV0Vlauqsobd5tPWz2AKwHhBXMEWsAH+rSbz0UZUfaMVj1CFrcNQK47ygmrC4vHmc7eOp1LdM+AStk5mMyA=="}`},
+ },
+ {
+ name: `connection_string_2`,
+ input: `EndpointSuffix=core.windows.net;AccountKey=ldlKgoKPJhRjPJTkaC5c/QNqtu4sVQRc/teGJ0MZHbDYEHdvBV5z8JEfJK+evE87D7U8TzMZ0G2C+ASt2B4ifg==;AccountName=storagetest123;DefaultEndpointsProtocol=http`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"ldlKgoKPJhRjPJTkaC5c/QNqtu4sVQRc/teGJ0MZHbDYEHdvBV5z8JEfJK+evE87D7U8TzMZ0G2C+ASt2B4ifg=="}`},
+ },
+ {
+ name: `connection_string_3`,
+ input: ` public const string SharedStorageKey = "DefaultEndpointsProtocol=https;AccountName=huntappstorage;AccountKey=rrttFty/b52ET/e8VqpMSN+ZqAUP7hcXVkdekrPX58gsMZyOCrE+igN07t3lyi7tAV0+OrJFBaDtMe06YJ2tFw==;EndpointSuffix=core.windows.net";`,
+ want: []string{`{"accountName":"huntappstorage","accountKey":"rrttFty/b52ET/e8VqpMSN+ZqAUP7hcXVkdekrPX58gsMZyOCrE+igN07t3lyi7tAV0+OrJFBaDtMe06YJ2tFw=="}`},
+ },
+ {
+ name: `connection_string_multiline`,
+ input: `
+export const DevelopmentConnectionString = 'DefaultEndpointsProtocol=http;AccountName=macdemostorage;
+ AccountKey=Jby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;
+ QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;';`,
+ want: []string{`{"accountName":"macdemostorage","accountKey":"Jby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="}`},
+ },
+
+ // LANGUAGES
+ // TODO:
+ // - https://github.com/Satyamk21/az204/blob/75f340c5bbfb34c1477a6885e216d5ae0972a380/Lab%203.txt#L22
+
+ // https://github.com/facebookincubator/velox/blob/98e958c0df498efd7cf44a2078cc71caeb7aed23/velox/connectors/hive/storage_adapters/abfs/tests/AzuriteServer.h#L32-L36
+ {
+ name: `cpp`,
+ input: `static const std::string AzuriteAccountName{"storagetest123"};
+static const std::string AzuriteContainerName{"test"};
+// the default key of Azurite Server used for connection
+static const std::string AzuriteAccountKey{
+ "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="};`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},
+ },
+ // https://github.com/MicrosoftDX/Dash/blob/03c4bb55f9e84fd03ee943559c128c4d5c2a31c2/DashServer.Tests/RequestAuthTests.cs#L29
+ {
+ name: `dotnet1`,
+ input: ` _ctx = InitializeConfigAndCreateTestBlobs(ctx, "datax1", new Dictionary<string, string>
+ {
+ { "AccountName", "dashstorage1" },
+ { "AccountKey", "8jqRVtXUWiEthgIhR+dFwrB8gh3lFuquvJQ1v4eabObIj7okI1cZIuzY8zZHmEdpcC0f+XlUkbFwAhjTfyrLIg==" },
+ { "SecondaryAccountKey", "Klari9ZbVdFQ35aULCfqqehCsd136amhusMHWynTpz2Pg+GyQMJw3GH177hvEQbaZ2oeRYk3jw0mIaV3ehNIRg==" },
+ },`,
+ want: []string{
+ `{"accountName":"dashstorage1","accountKey":"8jqRVtXUWiEthgIhR+dFwrB8gh3lFuquvJQ1v4eabObIj7okI1cZIuzY8zZHmEdpcC0f+XlUkbFwAhjTfyrLIg=="}`,
+ `{"accountName":"dashstorage1","accountKey":"Klari9ZbVdFQ35aULCfqqehCsd136amhusMHWynTpz2Pg+GyQMJw3GH177hvEQbaZ2oeRYk3jw0mIaV3ehNIRg=="}`,
+ },
+ },
+ // https://github.com/Satyamk21/az204/blob/75f340c5bbfb34c1477a6885e216d5ae0972a380/Lab%203.txt#L11
+ {
+ name: `dotnet2`,
+ input: `public class Program
+{
+ private const string blobServiceEndpoint = "https://k21storagemedia.blob.core.windows.net/";
+
+ private const string storageAccountName = "k21storagemedia";
+
+ private const string storageAccountKey = "DFdukxfl0SwO4NB91bi/FTPh9BMEKr6Z5wWf+tGDfXMakXvGVp/NDzAUjWc/9171OqoDvXSj1o8N+AStUk1GXg==";
+
+
+ //The following code to create a new asynchronous Main method
+ public static async Task Main(string[] args)`,
+ want: []string{`{"accountName":"k21storagemedia","accountKey":"DFdukxfl0SwO4NB91bi/FTPh9BMEKr6Z5wWf+tGDfXMakXvGVp/NDzAUjWc/9171OqoDvXSj1o8N+AStUk1GXg=="}`},
+ },
+ // https://github.com/apache/camel/blob/main/test-infra/camel-test-infra-azure-common/src/test/java/org/apache/camel/test/infra/azure/common/services/AzuriteContainer.java#L25-L27
+ {
+ name: `java`,
+ input: `public class AzuriteContainer extends GenericContainer<AzuriteContainer> {
+ public static final String DEFAULT_ACCOUNT_NAME = "storagetest123";
+ public static final String DEFAULT_ACCOUNT_KEY
+ = "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==";
+
+ public static final String IMAGE_NAME = "mcr.microsoft.com/azure-storage/azurite:3.27.0";`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},
+ },
+ // https://github.com/Azure/azure-storage-node/blob/6873387fc65bad6d577babe278be2ee2e6071493/test/common/connectionstringparsertests.js
+ {
+ name: `javascript`,
+ input: ` var parsedConnectionString = ServiceSettings.parseAndValidateKeys(defaultConnectionString + endpointsConnectionString, validKeys);
+ assert.equal(parsedConnectionString['DefaultEndpointsProtocol'], 'https');
+ assert.equal(parsedConnectionString['AccountName'], 'storagetest123');
+ assert.equal(parsedConnectionString['AccountKey'], 'KWPLd0rpW2T0U7K2pVpF8rYr1BgYtR7wYQk33AYiXeUoquiaY6o0TWqduxmPHlqeCNZ3LU0DHptbeIHy5l/Yhg==');
+ assert.equal(parsedConnectionString['BlobEndpoint'], 'myBlobEndpoint');
+ assert.equal(parsedConnectionString['QueueEndpoint'], 'myQueueEndpoint');
+ assert.equal(parsedConnectionString['TableEndpoint'], 'myTableEndpoint');`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"KWPLd0rpW2T0U7K2pVpF8rYr1BgYtR7wYQk33AYiXeUoquiaY6o0TWqduxmPHlqeCNZ3LU0DHptbeIHy5l/Yhg=="}`},
+ },
+ // https://github.com/nextcloud/server/blob/81a9e19ace190ea0a64d52d95d341e25c7ad618b/tests/preseed-config.php#L89
+ {
+ name: `php`,
+ input: ` 'arguments' => [
+ 'container' => 'test',
+ 'account_name' => 'storagetest123',
+ 'account_key' => 'qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==',
+ 'endpoint' => 'http://' . (getenv('DRONE') === 'true' ? 'azurite' : 'localhost') . ':10000/devstoreaccount1',
+ 'autocreate' => true
+ ]`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},
+ },
+ // https://github.com/Azure/azure-sdk-for-js/blob/2719dcfbe835a2da3003876dcb5d77efba95f912/sdk/cosmosdb/cosmos/test/public/common/_fakeTestSecrets.ts
+ {
+ name: `typescript`,
+ input: `export const name =
+ process.env.ACCOUNT_NAME || "storagename123";
+export const key =
+ process.env.ACCOUNT_KEY ||
+ "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==";`,
+ want: []string{`{"accountName":"storagename123","accountKey":"C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="}`},
+ },
+
+ // FORMATS
+ // TODO: Doesn't work.
+ // https://github.com/Azure/azure-quickstart-templates/blob/03e792429fbc65c9353335611933746364590b22/quickstarts/microsoft.datafactory/data-factory-hive-transformation/azuredeploy.parameters.json#L9C38-L9C38
+ // {
+ // name: `json`,
+ // input: ` "storageAccountName": {
+ // "value": "changemeazurestorage"
+ //},
+ //"storageAccountKey": {
+ // "value": "YA1gKAMY34PeVgEWPF8FdbQO+U0nFkd3SaFE4d32K16AYL/DowrTYun8anOdAiCnMkCiRYm+PxUh5mw7a7lVcA=="
+ //},`,
+ // want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},
+ // },
+ // https://github.com/ClickHouse/ClickHouse/blob/eba52b318d67d85330c9c1781499b7ff27fb7c0e/tests/integration/test_storage_azure_blob_storage/configs/named_collections.xml
+ {
+ name: `xml`,
+ input: ` <azure_conf2>
+ <account_name>storagetest123</account_name>
+ <account_key>qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==</account_key>
+ </azure_conf2>`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},
+ },
+ // https://github.com/hubblestack/hubble/blob/f9b7bf38752bd16b27d050a3b8787652a1c6319b/hubblestack/fileserver/azurefs.py
+ {
+ name: `yaml1`,
+ input: ` azurefs:
+ - account_name: mystorage
+ account_key: 'fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA=='
+ container_name: my_container
+ proxy: 10.10.10.10:8080`,
+ want: []string{`{"accountName":"mystorage","accountKey":"fNH9cRp0+qVIVYZ+5rnZAhHc9ycOUcJnHtzpfOr0W0sxrtL2KVLuMe1xDfLwmfed+JJInZaEdWVCPHD4d/oqeA=="}`},
+ },
+ {
+ name: `yaml2`,
+ input: ` - name: filesharevolume
+ azureFile:
+ sharename: containershare
+ storageAccountName: newstore100033323
+ storageAccountKey: Ar4/2iY8L0rEMeQaijINnfaMJr7vqjfbPgmJayw6Pu5l9ZI+GrFDm1uIWOqXk5RQLrTiXfBwWY6hAbPEIQqy1g==`,
+ want: []string{`{"accountName":"newstore100033323","accountKey":"Ar4/2iY8L0rEMeQaijINnfaMJr7vqjfbPgmJayw6Pu5l9ZI+GrFDm1uIWOqXk5RQLrTiXfBwWY6hAbPEIQqy1g=="}`},
+ },
+ // This was manually base64-decoded since that doesn't work in unit tests.
+ // https://github.com/fabric8io/configmapcontroller/blob/master/vendor/k8s.io/kubernetes/examples/azure_file/secret/azure-secret.yaml
+ {
+ name: `yaml_3`,
+ input: `apiVersion: v1
+ kind: Secret
+ metadata:
+ name: azure-secret
+ type: Opaque
+ data:
+ azurestorageaccountname: k8stest
+ azurestorageaccountkey: xIF1zJbnnojFLMSkBp50mx02rHsMK2sjU7mFt4L13hoB7drAaJ8jD6+A443jJogV7y2FUOhQCWPmM6YaNHy7qg==
+`,
+ want: []string{`{"accountName":"k8stest","accountKey":"xIF1zJbnnojFLMSkBp50mx02rHsMK2sjU7mFt4L13hoB7drAaJ8jD6+A443jJogV7y2FUOhQCWPmM6YaNHy7qg=="}`},
+ },
+
+ // MISC
+ // https://github.com/Azure-Samples/nested-virtualization-image-builder/blob/cf0373a421343b00ce3d261be99ddced80deb55b/README.md?plain=1#L54
+ {
+ name: `blob_url`,
+ input: `"name": "storagetest123.blob.core.windows.net", "accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`},
+ },
+ {
+ name: `random_cli_1`,
+ input: `go run .\main.go -debug -dest="https://kenfau.blob.core.windows.net/ss3/" -AzureDefaultAccountName="kenfoo" -AzureDefaultAccountKey="hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="`,
+ want: []string{
+ `{"accountName":"kenfau","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`,
+ `{"accountName":"kenfoo","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`,
+ },
+ },
+ // - https://github.com/nwoolls/AzureStorageCleanup/blob/980e5cb163c78e9446e70d2513ba5a7ed9051a7a/README.md?plain=1#L24
+ {
+ name: `random_cli_2`,
+ input: `AzureStorageCleanup.exe
+ -storagename storageaccount
+ -storagekey hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w==
+ -container sqlbackup
+ -mindaysold 60
+ -searchpattern .*
+ -recursive
+ -whatif`,
+ want: []string{`{"accountName":"storageaccount","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`},
+ },
+ // https://github.com/dahlej/rpi-spark-titantic/blob/d00b8f5b4696aeb2113e9452c24bb31b7f9a0242/tmp.txt#L9
+ {
+ name: `random_cli_3`,
+ input: `$ bin/spark-submit --master \
+ k8s://test-cluster.eastus2.cloudapp.azure.com:443 \
+ --deploy-mode cluster \
+ --name copyLocations \
+ --class io.timpark.CopyData \
+ --conf spark.copydata.containerpath=wasb://[email protected] \
+ --conf spark.copydata.storageaccount=storagetest123 \
+ --conf spark.copydata.storageaccountkey=hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w== \`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"hGeB3WqDyx0mGsQMsQDl+gmnXa51ZODiBtcXJpMoRhPjkDm79f9ErNfaYizXm7nkElix8n2uBwNk6KY8Rc866w=="}`},
+ },
+ {
+ name: `custom_config_1`,
+ input: `driver := ArtifactDriver{
+ AccountName: "storagetest123",
+ AccountKey: "qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA==",
+ Container: "test",
+ }`,
+ want: []string{`{"accountName":"storagetest123","accountKey":"qYaZm3m8+Z2aYAiSDzzvStkTUgZXl29U76lDJ0qiob7bbV4g7kjtwO+FI2QoptGdZgEdtsAYzG8T0hl5TeftWA=="}`},
+ },
+ // https://github.com/MicrosoftDX/Dash/blob/master/LoadTestDotNet/GetBlobCoded.cs
+ {
+ name: `storage_account_1`,
+ input: ` this.Context.Add("StorageEndPoint", "http://dashstorage3.blob.core.windows.net");
+ this.Context.Add("StorageAccount", "dashstorage3");
+ this.Context.Add("AccountKey", "TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ==");
+ this.Context.Add("SendChunked", false);`,
+ want: []string{`{"accountName":"dashstorage3","accountKey":"TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="}`},
+ },
+ // https://github.com/kubecost/poc-common-configurations/blob/d626a48824a104e3089fc66ef57029f1e2212f6a/keys.txt#L18
+ {
+ name: `storage_account_2`,
+ input: `AZ_cloud_integration_subscriptionId:0bd50fdf-c923-4e1e-850c-196ddSAMPLE
+AZ_cloud_integration_azureStorageAccount:kubecostexport
+AZ_cloud_integration_azureStorageAccessKey:TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ==
+AZ_cloud_integration_azureStorageContainer:costexports`,
+ want: []string{`{"accountName":"kubecostexport","accountKey":"TP+G/9FTZRP1he1EpKilMercxSbMyqtaI9xTbc/3HqT2/FkxyIk1wVlBdemDFuYKStmlkFqHc7049l8McTd8NQ=="}`},
+ },
+
+ // False positives
+ {
+ name: `test_key`,
+ input: ` azureblockblob: TEST_BACKEND=azureblockblob://DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;`,
+ },
+ {
+ name: `test_key_multiline`,
+ input: `
+export const DevelopmentConnectionString = 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;
+ AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;
+ QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;';`,
+ },
+ {
+ name: `invalid_key_1`,
+ input: ` docs::examples = "DefaultEndpointsProtocol=https;AccountName=mylogstorage;AccountKey=storageaccountkeybase64encoded;EndpointSuffix=core.windows.net"`,
+ },
+ {
+ name: `invalid_key_2`,
+ input: `PS C:\> Add-AzIotHubRoutingEndpoint -ResourceGroupName "myresourcegroup" -Name "myiothub" -EndpointName S1 -EndpointType AzureStorageContainer -EndpointResourceGroup resourcegroup1 -EndpointSubscriptionId 91d12343-a3de-345d-b2ea-135792468abc -ConnectionString 'DefaultEndpointsProtocol=https;AccountName=mystorage1;AccountKey=*****;EndpointSuffix=core.windows.net' -ContainerName container -Encoding json`,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ if diff := cmp.Diff(test.want, results); diff != "" {
+ t.Errorf("%s expected %d results, received %d: (-want +got)\n%s", test.name, len(test.want), len(results), diff)
+ }
+ }
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/azurestorage/azurestorage.go b/pkg/detectors/azurestorage/azurestorage.go
deleted file mode 100644
index 62797e361..000000000
--- a/pkg/detectors/azurestorage/azurestorage.go
+++ /dev/null
@@ -1,146 +0,0 @@
-package azurestorage
-
-import (
- "context"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/base64"
- "encoding/xml"
- "fmt"
- "io"
- "net/http"
- "strings"
- "time"
-
- regexp "github.com/wasilibs/go-re2"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct {
- client *http.Client
-}
-
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
- defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
- keyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P<account_name>[^;]+);AccountKey=(?P<account_key>[^;]+);EndpointSuffix=core\.windows\.net`)
-)
-
-type storageResponse struct {
- Containers struct {
- Container []container `xml:"Container"`
- } `xml:"Containers"`
-}
-
-type container struct {
- Name string `xml:"Name"`
-}
-
-func (s Scanner) Keywords() []string {
- return []string{"DefaultEndpointsProtocol=https;AccountName="}
-}
-
-func (s Scanner) getClient() *http.Client {
- if s.client != nil {
- return s.client
- }
- return defaultClient
-}
-
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, match := range matches {
- if len(match) != 3 {
- continue
- }
- accountName := strings.TrimSpace(match[1])
- accountKey := strings.TrimSpace(match[2])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_AzureStorage,
- Raw: []byte(accountKey),
- ExtraData: map[string]string{
- "account_name": accountName,
- },
- }
-
- if verify {
- client := s.getClient()
-
- isVerified, verificationErr := verifyAzureStorageKey(ctx, client, accountName, accountKey, s1.ExtraData)
- s1.Verified = isVerified
- s1.SetVerificationError(verificationErr, accountKey)
- }
-
- results = append(results, s1)
- }
-
- return results, nil
-}
-
-func verifyAzureStorageKey(ctx context.Context, client *http.Client, accountName, accountKey string, extraData map[string]string) (bool, error) {
- now := time.Now().UTC().Format(http.TimeFormat)
- stringToSign := "GET\n\n\n\n\n\n\n\n\n\n\n\nx-ms-date:" + now + "\nx-ms-version:2019-12-12\n/" + accountName + "/\ncomp:list"
- accountKeyBytes, _ := base64.StdEncoding.DecodeString(accountKey)
- h := hmac.New(sha256.New, accountKeyBytes)
- h.Write([]byte(stringToSign))
- signature := base64.StdEncoding.EncodeToString(h.Sum(nil))
-
- url := "https://" + accountName + ".blob.core.windows.net/?comp=list"
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
- if err != nil {
- return false, err
- }
- req.Header.Set("x-ms-date", now)
- req.Header.Set("x-ms-version", "2019-12-12")
- req.Header.Set("Authorization", "SharedKey "+accountName+":"+signature)
-
- res, err := client.Do(req)
- if err != nil {
- // If the host is not found, we can assume that the accountName is not valid
- if strings.Contains(err.Error(), "no such host") {
- return false, nil
- }
- return false, err
- }
- defer res.Body.Close()
-
- switch res.StatusCode {
- case http.StatusOK:
- body, err := io.ReadAll(res.Body)
- // ignore the error if failed because our keys has been verified.
- if err == nil {
- // parse response
- response := storageResponse{}
- if err := xml.Unmarshal(body, &response); err == nil {
- // update the extra data with container names only
- var containerNames []string
- for _, c := range response.Containers.Container {
- containerNames = append(containerNames, c.Name)
- }
- extraData["container_names"] = strings.Join(containerNames, ", ")
- }
- }
-
- return true, nil
- case http.StatusForbidden:
- // 403 if account id or key is invalid, or if the account is disabled
- return false, nil
- default:
- return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
- }
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
- return detectorspb.DetectorType_AzureStorage
-}
-
-func (s Scanner) Description() string {
- return "Azure Storage is a Microsoft-managed cloud service that provides storage that is highly available, secure, durable, scalable, and redundant. Azure Storage Account keys can be used to access and manage data within storage accounts."
-}
diff --git a/pkg/detectors/azurestorage/azurestorage_test.go b/pkg/detectors/azurestorage/azurestorage_test.go
deleted file mode 100644
index 676900289..000000000
--- a/pkg/detectors/azurestorage/azurestorage_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package azurestorage
-
-import (
- "context"
- "testing"
-
- "github.com/google/go-cmp/cmp"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
-)
-
-var (
- validPattern = `
- azure storage:
- DefaultEndpointsProtocol=https;AccountName=myStorageAccount;AccountKey=myAccountKey12345==;EndpointSuffix=core.windows.net
- `
- invalidPattern = `
- azure storage:
- DefaultEndpointsProtocol=https;AccountName=;AccountKey=myAccountKey12345==;EndpointSuffix=core.ubuntu.net
- `
-)
-
-func TestAzureStorage_Pattern(t *testing.T) {
- d := Scanner{}
- ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
-
- tests := []struct {
- name string
- input string
- want []string
- }{
- {
- name: "valid pattern",
- input: validPattern,
- want: []string{"myAccountKey12345=="},
- },
- {
- name: "invalid pattern",
- input: invalidPattern,
- want: nil,
- },
- }
-
- for _, test := range tests {
- t.Run(test.name, func(t *testing.T) {
- matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
- if len(matchedDetectors) == 0 {
- t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
- return
- }
-
- results, err := d.FromData(context.Background(), false, []byte(test.input))
- if err != nil {
- t.Errorf("error = %v", err)
- return
- }
-
- if len(results) != len(test.want) {
- if len(results) == 0 {
- t.Errorf("did not receive result")
- } else {
- t.Errorf("expected %d results, only received %d", len(test.want), len(results))
- }
- return
- }
-
- actual := make(map[string]struct{}, len(results))
- for _, r := range results {
- if len(r.RawV2) > 0 {
- actual[string(r.RawV2)] = struct{}{}
- } else {
- actual[string(r.Raw)] = struct{}{}
- }
- }
- expected := make(map[string]struct{}, len(test.want))
- for _, v := range test.want {
- expected[v] = struct{}{}
- }
-
- if diff := cmp.Diff(expected, actual); diff != "" {
- t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
- }
- })
- }
-}
diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go
index 888fbb2d9..ff4a83b13 100644
--- a/pkg/engine/defaults/defaults.go
+++ b/pkg/engine/defaults/defaults.go
@@ -73,11 +73,11 @@ import (
azure_serviceprincipal_v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v1"
azure_serviceprincipal_v2 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_entra/serviceprincipal/v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_openai"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure_storage"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurecontainerregistry"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuredevopspersonalaccesstoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchadminkey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azuresearchquerykey"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azurestorage"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baremetrics"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beamer"
@@ -899,7 +899,7 @@ func buildDetectorList() []detectors.Detector {
&azure_openai.Scanner{},
&azuresearchadminkey.Scanner{},
&azuresearchquerykey.Scanner{},
- &azurestorage.Scanner{},
+ &azure_storage.Scanner{},
&bannerbear.Scanner{},
&baremetrics.Scanner{},
&beamer.Scanner{},