Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
detector improvements
dmarquero0fc7964c3f64
diff --git a/pkg/detectors/abuseipdb/abuseipdb.go b/pkg/detectors/abuseipdb/abuseipdb.goindex 3c5132d6e..abc19c766 100644--- a/pkg/detectors/abuseipdb/abuseipdb.go+++ b/pkg/detectors/abuseipdb/abuseipdb.go@@ -3,10 +3,13 @@ package abuseipdbimport ("context""io/ioutil"- "net/http"++ // "log""regexp""strings"+ "net/http"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -55,14 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)+ validResponse := strings.Contains(bodyString, `ipAddress`)+ //errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/allsports/allsports.go b/pkg/detectors/allsports/allsports.goindex 131379a33..09f8094d5 100644--- a/pkg/detectors/allsports/allsports.go+++ b/pkg/detectors/allsports/allsports.go@@ -3,10 +3,13 @@ package allsportsimport ("context""io/ioutil"- "net/http"++ // "log""regexp""strings"+ "net/http"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -55,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if !strings.Contains(body, "Wrong login credentials") {+ if strings.Contains(body, "success") {s1.Verified = true} else {if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {diff --git a/pkg/detectors/apptivo/apptivo.go b/pkg/detectors/apptivo/apptivo.goindex 55ab7bf6e..92fa4bec4 100644--- a/pkg/detectors/apptivo/apptivo.go+++ b/pkg/detectors/apptivo/apptivo.go@@ -64,13 +64,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"status":"FAILURE"`)+ validResponse := strings.Contains(bodyString, `displayName`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/audd/audd.go b/pkg/detectors/audd/audd.goindex b8eb4d14f..16f2f64f6 100644--- a/pkg/detectors/audd/audd.go+++ b/pkg/detectors/audd/audd.go@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"status":"error"`)+ validResponse := strings.Contains(bodyString, `"status":"success"`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/baseapiio/baseapiio.go b/pkg/detectors/baseapiio/baseapiio.goindex edbace5ff..7e461eb9f 100644--- a/pkg/detectors/baseapiio/baseapiio.go+++ b/pkg/detectors/baseapiio/baseapiio.go@@ -1,11 +1,9 @@package baseapiioimport (- "bytes""context""fmt""io/ioutil"- "mime/multipart""net/http""regexp""strings"@@ -51,11 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- payload := &bytes.Buffer{}- writer := multipart.NewWriter(payload)- _ = writer.WriteField("name", "\"My Form\"")- req, _ := http.NewRequestWithContext(ctx, "POST", "https://api.base-api.io/v1/forms", payload)- req.Header.Set("Content-Type", writer.FormDataContentType())+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.base-api.io/v1/forms?page=1&per_page=10", nil)req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))res, err := client.Do(req)if err == nil {@@ -63,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if !strings.Contains(body, "UNAUTHENTICATED") {+ if strings.Contains(body, "items") {s1.Verified = true} else {if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {diff --git a/pkg/detectors/besttime/besttime.go b/pkg/detectors/besttime/besttime.goindex 6ea57d773..ee21a8764 100644--- a/pkg/detectors/besttime/besttime.go+++ b/pkg/detectors/besttime/besttime.go@@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if !strings.Contains(body, "Invalid api_key_private") {+ if strings.Contains(body, `"status": "OK"`) {s1.Verified = true} else {if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {diff --git a/pkg/detectors/bulbul/bulbul.go b/pkg/detectors/bulbul/bulbul.goindex c5c31c22c..cb5dccff4 100644--- a/pkg/detectors/bulbul/bulbul.go+++ b/pkg/detectors/bulbul/bulbul.go@@ -52,20 +52,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreq, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://prod-api.bulbul.io/view_all_users?api_key=%s", resMatch), nil)res, err := client.Do(req)if err == nil {- bodyBytes, err := ioutil.ReadAll(res.Body)- if err == nil {- continue- }+ bodyBytes, _ := ioutil.ReadAll(res.Body)bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"status":102`)+ validResponse := strings.Contains(bodyString, `"message":"Successful",`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/commodities/commodities.go b/pkg/detectors/commodities/commodities.goindex ded71a042..5b4720dd9 100644--- a/pkg/detectors/commodities/commodities.go+++ b/pkg/detectors/commodities/commodities.go@@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"type":"invalid_access_key"`)+ validResponse := strings.Contains(bodyString, `"success":true`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/convier/convier.go b/pkg/detectors/convier/convier.goindex 70e4e5128..32d50ce43 100644--- a/pkg/detectors/convier/convier.go+++ b/pkg/detectors/convier/convier.go@@ -61,13 +61,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}bodyString := string(bodyBytes)- errCode := !strings.Contains(bodyString, `"error":false`)+ validResponse := strings.Contains(bodyString, `"error":false`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/cryptocompare/cryptocompare.go b/pkg/detectors/cryptocompare/cryptocompare.goindex 37c6ae1ea..73a4f24c0 100644--- a/pkg/detectors/cryptocompare/cryptocompare.go+++ b/pkg/detectors/cryptocompare/cryptocompare.go@@ -56,13 +56,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"Response":"Error"`)+ errCode := strings.Contains(bodyString, `"Response":"Success"`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {if errCode {- s1.Verified = false- } else {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/diffbot/diffbot.go b/pkg/detectors/diffbot/diffbot.goindex d58427cea..1635d3eff 100644--- a/pkg/detectors/diffbot/diffbot.go+++ b/pkg/detectors/diffbot/diffbot.go@@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"errorCode": 401`)+ validResponse := strings.Contains(bodyString, `"token":`) && strings.Contains(bodyString, `"planCredits":`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/dynalist/dynalist.go b/pkg/detectors/dynalist/dynalist.goindex 36155c105..ba66a08b4 100644--- a/pkg/detectors/dynalist/dynalist.go+++ b/pkg/detectors/dynalist/dynalist.go@@ -57,13 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"_code":"InvalidToken"`)+ validResponse := strings.Contains(bodyString, `"_code":"Ok"`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/fakejson/fakejson.go b/pkg/detectors/fakejson/fakejson.goindex bec4369cb..9ae16fb24 100644--- a/pkg/detectors/fakejson/fakejson.go+++ b/pkg/detectors/fakejson/fakejson.go@@ -57,13 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"error":`)+ validResponse := strings.Contains(bodyString, `"user_email":`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/fetchrss/fetchrss_test.go b/pkg/detectors/fetchrss/fetchrss_test.goindex 59a7588c4..1e9bff2fe 100644--- a/pkg/detectors/fetchrss/fetchrss_test.go+++ b/pkg/detectors/fetchrss/fetchrss_test.go@@ -16,7 +16,7 @@ import (func TestFetchrss_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/flickr/flickr.go b/pkg/detectors/flickr/flickr.goindex 58f09d20b..d51f92215 100644--- a/pkg/detectors/flickr/flickr.go+++ b/pkg/detectors/flickr/flickr.go@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if (res.StatusCode >= 200 && res.StatusCode < 300) && !strings.Contains(body, "Invalid API Key") {+ if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "owner=") {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.goindex 89c4d8443..71853fa0e 100644--- a/pkg/detectors/freshbooks/freshbooks.go+++ b/pkg/detectors/freshbooks/freshbooks.go@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Log In To Your Accountant Center") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "Log In to FreshBooks") {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/freshbooks/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks/freshbooks.godeleted file mode 100644index 89c4d8443..000000000--- a/pkg/detectors/freshbooks/freshbooks/freshbooks.go+++ /dev/null@@ -1,81 +0,0 @@-package freshbooks--import (- "context"- "fmt"- "io/ioutil"- "net/http"- "regexp"- "strings"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--type Scanner struct{}--// Ensure the Scanner satisfies the interface at compile time-var _ detectors.Detector = (*Scanner)(nil)--var (- client = common.SaneHttpClient()-- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b([0-9a-z]{64})\b`)- uriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`)-)--// Keywords are used for efficiently pre-filtering chunks.-// Use identifiers in the secret preferably, or the provider name.-func (s Scanner) Keywords() []string {- return []string{"freshbooks"}-}--// FromData will find and optionally verify Freshbooks secrets in a given set of bytes.-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- dataStr := string(data)-- matches := keyPat.FindAllStringSubmatch(dataStr, -1)- uriMatches := uriPat.FindAllStringSubmatch(dataStr, -1)-- for _, match := range matches {- if len(match) != 2 {- continue- }- resMatch := strings.TrimSpace(match[1])- for _, uriMatch := range uriMatches {- if len(uriMatch) != 2 {- continue- }- resURI := strings.TrimSpace(uriMatch[1])-- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_Freshbooks,- Raw: []byte(resMatch),- }-- if verify {- req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(`https://auth.freshbooks.com/oauth/authorize?client_id=%s&redirect_uri=%s&response_type=code`, resMatch, resURI), nil)- res, err := client.Do(req)- if err == nil {- defer res.Body.Close()- bodyBytes, _ := ioutil.ReadAll(res.Body)- body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Log In To Your Accountant Center") {- s1.Verified = true- } else {- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }- }- }- }-- results = append(results, s1)- }- }-- return detectors.CleanResults(results), nil-}diff --git a/pkg/detectors/freshbooks/freshbooks/freshbooks_test.go b/pkg/detectors/freshbooks/freshbooks/freshbooks_test.godeleted file mode 100644index 53c41c5dc..000000000--- a/pkg/detectors/freshbooks/freshbooks/freshbooks_test.go+++ /dev/null@@ -1,114 +0,0 @@-package freshbooks--import (- "context"- "fmt"- "testing"- "time"-- "github.com/kylelemons/godebug/pretty"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--func TestFreshbooks_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("FRESHBOOKS")- uri := testSecrets.MustGetField("REDIRECT_URI")- inactiveSecret := testSecrets.MustGetField("FRESHBOOKS_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a freshbooks secret %s within freshbooks %s", secret, uri)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Freshbooks,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a freshbooks secret %s within freshbooks %s but not valid", inactiveSecret, uri)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Freshbooks,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Freshbooks.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Freshbooks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)- }- })- }-}diff --git a/pkg/detectors/geocode/geocode.go b/pkg/detectors/geocode/geocode.goindex ca128c815..229eabbfa 100644--- a/pkg/detectors/geocode/geocode.go+++ b/pkg/detectors/geocode/geocode.go@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"code" : "003"`)+ validResponse := strings.Contains(bodyString, `"remaining_credits" :`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/ipinfodb/ipinfodb.go b/pkg/detectors/ipinfodb/ipinfodb.goindex a33e9f290..851118a78 100644--- a/pkg/detectors/ipinfodb/ipinfodb.go+++ b/pkg/detectors/ipinfodb/ipinfodb.go@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"statusMessage" : "Invalid API key."`)+ validResponse := strings.Contains(bodyString, `"statusCode": "OK"`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/linearapi/linearapi.go b/pkg/detectors/linearapi/linearapi.goindex ffa333992..68af1f5fd 100644--- a/pkg/detectors/linearapi/linearapi.go+++ b/pkg/detectors/linearapi/linearapi.go@@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "authentication failed") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"data":`) {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/metaapi/metaapi.go b/pkg/detectors/metaapi/metaapi.goindex 7482e5f83..a095bb2ba 100644--- a/pkg/detectors/metaapi/metaapi.go+++ b/pkg/detectors/metaapi/metaapi.go@@ -68,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif errBody == nil {bodyStr := string(body)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(bodyStr, "<style>") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(bodyStr, `"success":true`) {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/midise/midise.go b/pkg/detectors/midise/midise.godeleted file mode 100644index a57446981..000000000--- a/pkg/detectors/midise/midise.go+++ /dev/null@@ -1,74 +0,0 @@-package midise--import (- "context"- "fmt"- "net/http"- "regexp"-- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"-)--type Scanner struct{}--// Ensure the Scanner satisfies the interface at compile time-var _ detectors.Detector = (*Scanner)(nil)--var (- secretKey = regexp.MustCompile(`midi-662b69edd2[a-zA-Z0-9]{54}`)-)--// Keywords are used for efficiently pre-filtering chunks.-// Use identifiers in the secret preferably, or the provider name.-func (s Scanner) Keywords() []string {- return []string{"midi-662b69edd2"}-}--// FromData will find and optionally verify Midise secrets in a given set of bytes.-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {-- dataStr := string(data)-- matches := secretKey.FindAllString(dataStr, -1)-- for _, match := range matches {-- s := detectors.Result{- DetectorType: detectorspb.DetectorType_Midise,- Raw: []byte(match),- }-- if verify {-- baseURL := "https://hooks.zapier.com/hooks/catch/11693304/bssgr3k"-- client := common.SaneHttpClient()-- req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match))- req.Header.Add("Content-Type", "application/json")- res, err := client.Do(req)- if err != nil {- return results, err- }- defer res.Body.Close()-- if res.StatusCode == http.StatusOK {- s.Verified = true- }- }-- if !s.Verified {- if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {- continue- }- }-- results = append(results, s)- }-- return-}diff --git a/pkg/detectors/midise/midise_test.go b/pkg/detectors/midise/midise_test.godeleted file mode 100644index 71c1d23da..000000000--- a/pkg/detectors/midise/midise_test.go+++ /dev/null@@ -1,70 +0,0 @@-package midise--import (- "context"- "testing"-- "github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"-)--func TestTruffle_FromChunk(t *testing.T) {- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- }{- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Midise.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatal("no raw secret present")- }- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Midise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- for n := 0; n < b.N; n++ {- _, err := s.FromData(ctx, false, data)- if err != nil {- b.Fatal(err)- }- }- })- }-}diff --git a/pkg/detectors/mindmeister/mindmeister.go b/pkg/detectors/mindmeister/mindmeister.goindex 8c27f8025..9503759c1 100644--- a/pkg/detectors/mindmeister/mindmeister.go+++ b/pkg/detectors/mindmeister/mindmeister.go@@ -56,13 +56,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"code":"1010"`)+ validResponse := strings.Contains(bodyString, `"stat":"ok"`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/mindmeister/mindmeister/mindmeister.go b/pkg/detectors/mindmeister/mindmeister/mindmeister.godeleted file mode 100644index 8c27f8025..000000000--- a/pkg/detectors/mindmeister/mindmeister/mindmeister.go+++ /dev/null@@ -1,81 +0,0 @@-package mindmeister--import (- "context"- "fmt"- "io/ioutil"- "net/http"- "regexp"- "strings"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--type Scanner struct{}--// Ensure the Scanner satisfies the interface at compile time-var _ detectors.Detector = (*Scanner)(nil)--var (- client = common.SaneHttpClient()-- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mindmeister"}) + `\b([a-zA-Z0-9]{43})\b`)-)--// Keywords are used for efficiently pre-filtering chunks.-// Use identifiers in the secret preferably, or the provider name.-func (s Scanner) Keywords() []string {- return []string{"mindmeister"}-}--// FromData will find and optionally verify Mindmeister secrets in a given set of bytes.-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- dataStr := string(data)-- matches := keyPat.FindAllStringSubmatch(dataStr, -1)-- for _, match := range matches {- if len(match) != 2 {- continue- }- resMatch := strings.TrimSpace(match[1])-- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_Mindmeister,- Raw: []byte(resMatch),- }-- if verify {- req, _ := http.NewRequestWithContext(ctx, "GET", "https://www.mindmeister.com/services/rest/oauth2?method=mm.maps.getList", nil)- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))- res, err := client.Do(req)- if err == nil {- bodyBytes, err := ioutil.ReadAll(res.Body)- if err == nil {- bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"code":"1010"`)- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {- s1.Verified = true- }- } else {- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }- }- }- }- }-- results = append(results, s1)- }-- return detectors.CleanResults(results), nil-}diff --git a/pkg/detectors/mindmeister/mindmeister/mindmeister_test.go b/pkg/detectors/mindmeister/mindmeister/mindmeister_test.godeleted file mode 100644index 1d4f939f1..000000000--- a/pkg/detectors/mindmeister/mindmeister/mindmeister_test.go+++ /dev/null@@ -1,113 +0,0 @@-package mindmeister--import (- "context"- "fmt"- "testing"- "time"-- "github.com/kylelemons/godebug/pretty"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--func TestMindmeister_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("MINDMEISTER")- inactiveSecret := testSecrets.MustGetField("MINDMEISTER_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a mindmeister secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Mindmeister,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a mindmeister secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Mindmeister,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Mindmeister.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Mindmeister.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)- }- })- }-}diff --git a/pkg/detectors/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock.goindex 0751fbef0..9b7ce5f50 100644--- a/pkg/detectors/mrticktock/mrticktock.go+++ b/pkg/detectors/mrticktock/mrticktock.go@@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Wrong credentials.") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"errors":[]`) {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/mrticktock/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock/mrticktock.godeleted file mode 100644index 0751fbef0..000000000--- a/pkg/detectors/mrticktock/mrticktock/mrticktock.go+++ /dev/null@@ -1,83 +0,0 @@-package mrticktock--import (- "context"- "fmt"- "io/ioutil"- "net/http"- "regexp"- "strings"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--type Scanner struct{}--// Ensure the Scanner satisfies the interface at compile time-var _ detectors.Detector = (*Scanner)(nil)--var (- client = common.SaneHttpClient()-- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives- emailPat = regexp.MustCompile(`\b([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-z]+)\b`)- pwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mrticktock"}) + `\b([a-zA-Z0-9!=@#$%()_^]{1,50})`)-)--// Keywords are used for efficiently pre-filtering chunks.-// Use identifiers in the secret preferably, or the provider name.-func (s Scanner) Keywords() []string {- return []string{"mrticktock"}-}--// FromData will find and optionally verify Mrticktock secrets in a given set of bytes.-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- dataStr := string(data)-- matches := emailPat.FindAllStringSubmatch(dataStr, -1)- passwordMatches := pwordPat.FindAllStringSubmatch(dataStr, -1)-- for _, match := range matches {- if len(match) != 2 {- continue- }- resMatch := strings.TrimSpace(match[1])-- for _, passwordMatch := range passwordMatches {- if len(passwordMatch) != 2 {- continue- }- resPassword := strings.TrimSpace(passwordMatch[1])-- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_Mrticktock,- Raw: []byte(resMatch),- }-- if verify {- payload := strings.NewReader(fmt.Sprintf(`email=%s&password=%s`, resMatch, resPassword))- req, _ := http.NewRequestWithContext(ctx, "POST", "https://mrticktock.com/app/api/is_timer_active", payload)- req.Header.Add("Content-Type", "application/x-www-form-urlencoded")- res, err := client.Do(req)- if err == nil {- defer res.Body.Close()- bodyBytes, _ := ioutil.ReadAll(res.Body)- body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Wrong credentials.") {- s1.Verified = true- } else {- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }- }- }- }-- results = append(results, s1)- }- }- return detectors.CleanResults(results), nil-}diff --git a/pkg/detectors/mrticktock/mrticktock/mrticktock_test.go b/pkg/detectors/mrticktock/mrticktock/mrticktock_test.godeleted file mode 100644index d3a7440ee..000000000--- a/pkg/detectors/mrticktock/mrticktock/mrticktock_test.go+++ /dev/null@@ -1,114 +0,0 @@-package mrticktock--import (- "context"- "fmt"- "testing"- "time"-- "github.com/kylelemons/godebug/pretty"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--func TestMrticktock_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- email := testSecrets.MustGetField("SCANNERS_EMAIL")- pword := testSecrets.MustGetField("SCANNERS_PASS")- inactiveSecret := testSecrets.MustGetField("SCANNERS_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a mrticktock secret %s within %s", email, pword)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Mrticktock,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a mrticktock secret %s within %s but not valid", email, inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Mrticktock,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("Mrticktock.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("Mrticktock.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)- }- })- }-}diff --git a/pkg/detectors/onedesk/onedesk.go b/pkg/detectors/onedesk/onedesk.goindex 77e6d0f45..18d9d9ffd 100644--- a/pkg/detectors/onedesk/onedesk.go+++ b/pkg/detectors/onedesk/onedesk.go@@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "ERROR_LOGIN_FAILED") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"code":"SUCCESS"`) {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/paralleldots/paralleldots.go b/pkg/detectors/paralleldots/paralleldots.goindex 3cf960713..a2b5cc27a 100644--- a/pkg/detectors/paralleldots/paralleldots.go+++ b/pkg/detectors/paralleldots/paralleldots.go@@ -77,7 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if (res.StatusCode >= 200 && res.StatusCode < 300) && !strings.Contains(body, "Invalid API Key") {+ if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/pdflayer/pdflayer.go b/pkg/detectors/pdflayer/pdflayer.goindex 0bf089f0c..0bd108ea4 100644--- a/pkg/detectors/pdflayer/pdflayer.go+++ b/pkg/detectors/pdflayer/pdflayer.go@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"code":101`)+ validResponse := strings.Contains(bodyString, `Contents`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/scrapersite/scrapersite.go b/pkg/detectors/scrapersite/scrapersite.goindex 804a52bf6..b63abdad4 100644--- a/pkg/detectors/scrapersite/scrapersite.go+++ b/pkg/detectors/scrapersite/scrapersite.go@@ -55,22 +55,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreq, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://scrapersite.com/api-v1?api_key=%s&url=https://google.com", resMatch), nil)res, err := client.Do(req)if err == nil {- bodyBytes, err := ioutil.ReadAll(res.Body)- if err == nil {- bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"code":101`)- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {- s1.Verified = true- }+ bodyBytes, _ := ioutil.ReadAll(res.Body)+ bodyString := string(bodyBytes)+ validResponse := strings.Contains(bodyString, `"status":true`)+ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ if validResponse {+ s1.Verified = true} else {- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }+ s1.Verified = false+ }+ } else {+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue}}}diff --git a/pkg/detectors/scrapersite/scrapersite/scrapersite.go b/pkg/detectors/scrapersite/scrapersite/scrapersite.godeleted file mode 100644index 632a9597d..000000000--- a/pkg/detectors/scrapersite/scrapersite/scrapersite.go+++ /dev/null@@ -1,86 +0,0 @@-package scrapersite--import (- "context"- "fmt"- "io/ioutil"- "net/http"- "regexp"- "strings"- "time"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--type Scanner struct{}--// Ensure the Scanner satisfies the interface at compile time-var _ detectors.Detector = (*Scanner)(nil)--var (- client = common.SaneHttpClient()-- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)-)--// Keywords are used for efficiently pre-filtering chunks.-// Use identifiers in the secret preferably, or the provider name.-func (s Scanner) Keywords() []string {- return []string{"scrapersite"}-}--// FromData will find and optionally verify ScraperSite secrets in a given set of bytes.-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- dataStr := string(data)-- matches := keyPat.FindAllStringSubmatch(dataStr, -1)-- for _, match := range matches {- if len(match) != 2 {- continue- }- resMatch := strings.TrimSpace(match[1])-- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_ScraperSite,- Raw: []byte(resMatch),- }-- if verify {- timeout := 10 * time.Second- client.Timeout = timeout- req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://scrapersite.com/api-v1?api_key=%s&url=https://google.com", resMatch), nil)- if err != nil {- continue- }- res, err := client.Do(req)- if err == nil {- bodyBytes, err := ioutil.ReadAll(res.Body)- if err == nil {- bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"code":101`)- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {- s1.Verified = true- }- } else {- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }- }- }- }- }-- results = append(results, s1)- }-- return detectors.CleanResults(results), nil-}diff --git a/pkg/detectors/scrapersite/scrapersite/scrapersite_test.go b/pkg/detectors/scrapersite/scrapersite/scrapersite_test.godeleted file mode 100644index fc0d02a65..000000000--- a/pkg/detectors/scrapersite/scrapersite/scrapersite_test.go+++ /dev/null@@ -1,113 +0,0 @@-package scrapersite--import (- "context"- "fmt"- "testing"- "time"-- "github.com/kylelemons/godebug/pretty"-- "github.com/trufflesecurity/trufflehog/v3/pkg/common"- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-)--func TestScraperSite_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("SCRAPERSITE")- inactiveSecret := testSecrets.MustGetField("SCRAPERSITE_INACTIVE")-- type args struct {- ctx context.Context- data []byte- verify bool- }- tests := []struct {- name string- s Scanner- args args- want []detectors.Result- wantErr bool- }{- {- name: "found, verified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapersite secret %s within", secret)),- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScraperSite,- Verified: true,- },- },- wantErr: false,- },- {- name: "found, unverified",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a scrapersite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation- verify: true,- },- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_ScraperSite,- Verified: false,- },- },- wantErr: false,- },- {- name: "not found",- s: Scanner{},- args: args{- ctx: context.Background(),- data: []byte("You cannot find the secret within"),- verify: true,- },- want: nil,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- s := Scanner{}- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)- if (err != nil) != tt.wantErr {- t.Errorf("ScraperSite.FromData() error = %v, wantErr %v", err, tt.wantErr)- return- }- for i := range got {- if len(got[i].Raw) == 0 {- t.Fatalf("no raw secret present: \n %+v", got[i])- }- got[i].Raw = nil- }- if diff := pretty.Compare(got, tt.want); diff != "" {- t.Errorf("ScraperSite.FromData() %s diff: (-got +want)\n%s", tt.name, diff)- }- })- }-}--func BenchmarkFromData(benchmark *testing.B) {- ctx := context.Background()- s := Scanner{}- for name, data := range detectors.MustGetBenchmarkData() {- benchmark.Run(name, func(b *testing.B) {- for n := 0; n < b.N; n++ {- s.FromData(ctx, false, data)- }- })- }-}diff --git a/pkg/detectors/semaphore/semaphore.go b/pkg/detectors/semaphore/semaphore.goindex d7c2669a7..2b1fd099c 100644--- a/pkg/detectors/semaphore/semaphore.go+++ b/pkg/detectors/semaphore/semaphore.go@@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "apikey is invalid") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "account_id") {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/upcdatabase/upcdatabase.go b/pkg/detectors/upcdatabase/upcdatabase.goindex 2da90b80a..825191bbb 100644--- a/pkg/detectors/upcdatabase/upcdatabase.go+++ b/pkg/detectors/upcdatabase/upcdatabase.go@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"error":{"message":"Your API Key is invalid.`)+ validResponse := strings.Contains(bodyString, `added_time`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/walkscore/walkscore.go b/pkg/detectors/walkscore/walkscore.goindex 75ee5df26..e3c5301a3 100644--- a/pkg/detectors/walkscore/walkscore.go+++ b/pkg/detectors/walkscore/walkscore.go@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && !strings.Contains(body, `Invalid 'wsapikey'`)) {+ if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, `distance`) {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/webscraping/webscraping.go b/pkg/detectors/webscraping/webscraping.goindex 80da03b3a..1836d455d 100644--- a/pkg/detectors/webscraping/webscraping.go+++ b/pkg/detectors/webscraping/webscraping.go@@ -48,14 +48,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.webscrapingapi.com/v1?api_key="+resMatch, nil)+ req, _ := http.NewRequestWithContext(ctx, "GET", `https://api.webscrapingapi.com/v1?api_key=`+resMatch, nil)res, err := client.Do(req)if err == nil {defer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if !strings.Contains(body, "Invalid API key") {+ if strings.Contains(body, "key `url` required.") {s1.Verified = true} else {if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {diff --git a/pkg/detectors/whoxy/whoxy.go b/pkg/detectors/whoxy/whoxy.goindex 6e01c8bc8..863340a65 100644--- a/pkg/detectors/whoxy/whoxy.go+++ b/pkg/detectors/whoxy/whoxy.go@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdefer res.Body.Close()bodyBytes, _ := ioutil.ReadAll(res.Body)body := string(bodyBytes)- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Incorrect API Key") {+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"status": 1`) {s1.Verified = true} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/worldcoinindex/worldcoinindex.go b/pkg/detectors/worldcoinindex/worldcoinindex.goindex 782ffc768..cf6f32d9f 100644--- a/pkg/detectors/worldcoinindex/worldcoinindex.go+++ b/pkg/detectors/worldcoinindex/worldcoinindex.go@@ -55,14 +55,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultbodyBytes, err := ioutil.ReadAll(res.Body)if err == nil {bodyString := string(bodyBytes)- errCode := strings.Contains(bodyString, `"error"`)+ validResponse := strings.Contains(bodyString, `"Markets"`)defer res.Body.Close()if res.StatusCode >= 200 && res.StatusCode < 300 {- if errCode {- s1.Verified = false- } else {+ if validResponse {s1.Verified = true+ } else {+ s1.Verified = false}} else {//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real keydiff --git a/pkg/detectors/zenscrape/zenscrape.go b/pkg/detectors/zenscrape/zenscrape.goindex 29fae3eed..9201dc207 100644--- a/pkg/detectors/zenscrape/zenscrape.go+++ b/pkg/detectors/zenscrape/zenscrape.go@@ -49,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- req, _ := http.NewRequestWithContext(ctx, "GET", "https://app.zenscrape.com/api/v1/get", nil)+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://app.zenscrape.com/api/v1/get?url=http://www.google.com", nil)req.Header.Add("apikey", fmt.Sprintf("%s", resMatch))res, err := client.Do(req)if err == nil {