Skip to content

Find, verify, and analyze leaked credentials

detector improvements

dmarquero0fc7964c3f64
diff --git a/pkg/detectors/abuseipdb/abuseipdb.go b/pkg/detectors/abuseipdb/abuseipdb.go
index 3c5132d6e..abc19c766 100644
--- a/pkg/detectors/abuseipdb/abuseipdb.go
+++ b/pkg/detectors/abuseipdb/abuseipdb.go
@@ -3,10 +3,13 @@ package abuseipdb
import (
"context"
"io/ioutil"
- "net/http"
+
+ // "log"
"regexp"
"strings"
+ "net/http"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -55,14 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)
+ validResponse := strings.Contains(bodyString, `ipAddress`)
+ //errCode := strings.Contains(bodyString, `AbuseIPDB APIv2 Server.`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/allsports/allsports.go b/pkg/detectors/allsports/allsports.go
index 131379a33..09f8094d5 100644
--- a/pkg/detectors/allsports/allsports.go
+++ b/pkg/detectors/allsports/allsports.go
@@ -3,10 +3,13 @@ package allsports
import (
"context"
"io/ioutil"
- "net/http"
+
+ // "log"
"regexp"
"strings"
+ "net/http"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -55,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if !strings.Contains(body, "Wrong login credentials") {
+ if strings.Contains(body, "success") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
diff --git a/pkg/detectors/apptivo/apptivo.go b/pkg/detectors/apptivo/apptivo.go
index 55ab7bf6e..92fa4bec4 100644
--- a/pkg/detectors/apptivo/apptivo.go
+++ b/pkg/detectors/apptivo/apptivo.go
@@ -64,13 +64,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"status":"FAILURE"`)
+ validResponse := strings.Contains(bodyString, `displayName`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/audd/audd.go b/pkg/detectors/audd/audd.go
index b8eb4d14f..16f2f64f6 100644
--- a/pkg/detectors/audd/audd.go
+++ b/pkg/detectors/audd/audd.go
@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"status":"error"`)
+ validResponse := strings.Contains(bodyString, `"status":"success"`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/baseapiio/baseapiio.go b/pkg/detectors/baseapiio/baseapiio.go
index edbace5ff..7e461eb9f 100644
--- a/pkg/detectors/baseapiio/baseapiio.go
+++ b/pkg/detectors/baseapiio/baseapiio.go
@@ -1,11 +1,9 @@
package baseapiio
import (
- "bytes"
"context"
"fmt"
"io/ioutil"
- "mime/multipart"
"net/http"
"regexp"
"strings"
@@ -51,11 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- payload := &bytes.Buffer{}
- writer := multipart.NewWriter(payload)
- _ = writer.WriteField("name", "\"My Form\"")
- req, _ := http.NewRequestWithContext(ctx, "POST", "https://api.base-api.io/v1/forms", payload)
- req.Header.Set("Content-Type", writer.FormDataContentType())
+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.base-api.io/v1/forms?page=1&per_page=10", nil)
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
@@ -63,7 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if !strings.Contains(body, "UNAUTHENTICATED") {
+ if strings.Contains(body, "items") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
diff --git a/pkg/detectors/besttime/besttime.go b/pkg/detectors/besttime/besttime.go
index 6ea57d773..ee21a8764 100644
--- a/pkg/detectors/besttime/besttime.go
+++ b/pkg/detectors/besttime/besttime.go
@@ -55,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if !strings.Contains(body, "Invalid api_key_private") {
+ if strings.Contains(body, `"status": "OK"`) {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
diff --git a/pkg/detectors/bulbul/bulbul.go b/pkg/detectors/bulbul/bulbul.go
index c5c31c22c..cb5dccff4 100644
--- a/pkg/detectors/bulbul/bulbul.go
+++ b/pkg/detectors/bulbul/bulbul.go
@@ -52,20 +52,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://prod-api.bulbul.io/view_all_users?api_key=%s", resMatch), nil)
res, err := client.Do(req)
if err == nil {
- bodyBytes, err := ioutil.ReadAll(res.Body)
- if err == nil {
- continue
- }
+ bodyBytes, _ := ioutil.ReadAll(res.Body)
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"status":102`)
+ validResponse := strings.Contains(bodyString, `"message":"Successful",`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/commodities/commodities.go b/pkg/detectors/commodities/commodities.go
index ded71a042..5b4720dd9 100644
--- a/pkg/detectors/commodities/commodities.go
+++ b/pkg/detectors/commodities/commodities.go
@@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"type":"invalid_access_key"`)
+ validResponse := strings.Contains(bodyString, `"success":true`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/convier/convier.go b/pkg/detectors/convier/convier.go
index 70e4e5128..32d50ce43 100644
--- a/pkg/detectors/convier/convier.go
+++ b/pkg/detectors/convier/convier.go
@@ -61,13 +61,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
bodyString := string(bodyBytes)
- errCode := !strings.Contains(bodyString, `"error":false`)
+ validResponse := strings.Contains(bodyString, `"error":false`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/cryptocompare/cryptocompare.go b/pkg/detectors/cryptocompare/cryptocompare.go
index 37c6ae1ea..73a4f24c0 100644
--- a/pkg/detectors/cryptocompare/cryptocompare.go
+++ b/pkg/detectors/cryptocompare/cryptocompare.go
@@ -56,13 +56,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"Response":"Error"`)
+ errCode := strings.Contains(bodyString, `"Response":"Success"`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
if errCode {
- s1.Verified = false
- } else {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/diffbot/diffbot.go b/pkg/detectors/diffbot/diffbot.go
index d58427cea..1635d3eff 100644
--- a/pkg/detectors/diffbot/diffbot.go
+++ b/pkg/detectors/diffbot/diffbot.go
@@ -58,13 +58,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"errorCode": 401`)
+ validResponse := strings.Contains(bodyString, `"token":`) && strings.Contains(bodyString, `"planCredits":`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/dynalist/dynalist.go b/pkg/detectors/dynalist/dynalist.go
index 36155c105..ba66a08b4 100644
--- a/pkg/detectors/dynalist/dynalist.go
+++ b/pkg/detectors/dynalist/dynalist.go
@@ -57,13 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"_code":"InvalidToken"`)
+ validResponse := strings.Contains(bodyString, `"_code":"Ok"`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/fakejson/fakejson.go b/pkg/detectors/fakejson/fakejson.go
index bec4369cb..9ae16fb24 100644
--- a/pkg/detectors/fakejson/fakejson.go
+++ b/pkg/detectors/fakejson/fakejson.go
@@ -57,13 +57,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"error":`)
+ validResponse := strings.Contains(bodyString, `"user_email":`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/fetchrss/fetchrss_test.go b/pkg/detectors/fetchrss/fetchrss_test.go
index 59a7588c4..1e9bff2fe 100644
--- a/pkg/detectors/fetchrss/fetchrss_test.go
+++ b/pkg/detectors/fetchrss/fetchrss_test.go
@@ -16,7 +16,7 @@ import (
func TestFetchrss_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/flickr/flickr.go b/pkg/detectors/flickr/flickr.go
index 58f09d20b..d51f92215 100644
--- a/pkg/detectors/flickr/flickr.go
+++ b/pkg/detectors/flickr/flickr.go
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if (res.StatusCode >= 200 && res.StatusCode < 300) && !strings.Contains(body, "Invalid API Key") {
+ if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "owner=") {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go
index 89c4d8443..71853fa0e 100644
--- a/pkg/detectors/freshbooks/freshbooks.go
+++ b/pkg/detectors/freshbooks/freshbooks.go
@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Log In To Your Accountant Center") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "Log In to FreshBooks") {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/freshbooks/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks/freshbooks.go
deleted file mode 100644
index 89c4d8443..000000000
--- a/pkg/detectors/freshbooks/freshbooks/freshbooks.go
+++ /dev/null
@@ -1,81 +0,0 @@
-package freshbooks
-
-import (
- "context"
- "fmt"
- "io/ioutil"
- "net/http"
- "regexp"
- "strings"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
- client = common.SaneHttpClient()
-
- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b([0-9a-z]{64})\b`)
- uriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
- return []string{"freshbooks"}
-}
-
-// FromData will find and optionally verify Freshbooks secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
- uriMatches := uriPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, match := range matches {
- if len(match) != 2 {
- continue
- }
- resMatch := strings.TrimSpace(match[1])
- for _, uriMatch := range uriMatches {
- if len(uriMatch) != 2 {
- continue
- }
- resURI := strings.TrimSpace(uriMatch[1])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_Freshbooks,
- Raw: []byte(resMatch),
- }
-
- if verify {
- req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(`https://auth.freshbooks.com/oauth/authorize?client_id=%s&redirect_uri=%s&response_type=code`, resMatch, resURI), nil)
- res, err := client.Do(req)
- if err == nil {
- defer res.Body.Close()
- bodyBytes, _ := ioutil.ReadAll(res.Body)
- body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Log In To Your Accountant Center") {
- s1.Verified = true
- } else {
- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
- continue
- }
- }
- }
- }
-
- results = append(results, s1)
- }
- }
-
- return detectors.CleanResults(results), nil
-}
diff --git a/pkg/detectors/freshbooks/freshbooks/freshbooks_test.go b/pkg/detectors/freshbooks/freshbooks/freshbooks_test.go
deleted file mode 100644
index 53c41c5dc..000000000
--- a/pkg/detectors/freshbooks/freshbooks/freshbooks_test.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package freshbooks
-
-import (
- "context"
- "fmt"
- "testing"
- "time"
-
- "github.com/kylelemons/godebug/pretty"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestFreshbooks_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("FRESHBOOKS")
- uri := testSecrets.MustGetField("REDIRECT_URI")
- inactiveSecret := testSecrets.MustGetField("FRESHBOOKS_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a freshbooks secret %s within freshbooks %s", secret, uri)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Freshbooks,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a freshbooks secret %s within freshbooks %s but not valid", inactiveSecret, uri)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Freshbooks,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Freshbooks.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Freshbooks.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
- }
- })
- }
-}
diff --git a/pkg/detectors/geocode/geocode.go b/pkg/detectors/geocode/geocode.go
index ca128c815..229eabbfa 100644
--- a/pkg/detectors/geocode/geocode.go
+++ b/pkg/detectors/geocode/geocode.go
@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"code" : "003"`)
+ validResponse := strings.Contains(bodyString, `"remaining_credits" :`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/ipinfodb/ipinfodb.go b/pkg/detectors/ipinfodb/ipinfodb.go
index a33e9f290..851118a78 100644
--- a/pkg/detectors/ipinfodb/ipinfodb.go
+++ b/pkg/detectors/ipinfodb/ipinfodb.go
@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"statusMessage" : "Invalid API key."`)
+ validResponse := strings.Contains(bodyString, `"statusCode": "OK"`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/linearapi/linearapi.go b/pkg/detectors/linearapi/linearapi.go
index ffa333992..68af1f5fd 100644
--- a/pkg/detectors/linearapi/linearapi.go
+++ b/pkg/detectors/linearapi/linearapi.go
@@ -58,7 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "authentication failed") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"data":`) {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/metaapi/metaapi.go b/pkg/detectors/metaapi/metaapi.go
index 7482e5f83..a095bb2ba 100644
--- a/pkg/detectors/metaapi/metaapi.go
+++ b/pkg/detectors/metaapi/metaapi.go
@@ -68,7 +68,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if errBody == nil {
bodyStr := string(body)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(bodyStr, "<style>") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(bodyStr, `"success":true`) {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/midise/midise.go b/pkg/detectors/midise/midise.go
deleted file mode 100644
index a57446981..000000000
--- a/pkg/detectors/midise/midise.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package midise
-
-import (
- "context"
- "fmt"
- "net/http"
- "regexp"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
- secretKey = regexp.MustCompile(`midi-662b69edd2[a-zA-Z0-9]{54}`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
- return []string{"midi-662b69edd2"}
-}
-
-// FromData will find and optionally verify Midise secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-
- dataStr := string(data)
-
- matches := secretKey.FindAllString(dataStr, -1)
-
- for _, match := range matches {
-
- s := detectors.Result{
- DetectorType: detectorspb.DetectorType_Midise,
- Raw: []byte(match),
- }
-
- if verify {
-
- baseURL := "https://hooks.zapier.com/hooks/catch/11693304/bssgr3k"
-
- client := common.SaneHttpClient()
-
- req, _ := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match))
- req.Header.Add("Content-Type", "application/json")
- res, err := client.Do(req)
- if err != nil {
- return results, err
- }
- defer res.Body.Close()
-
- if res.StatusCode == http.StatusOK {
- s.Verified = true
- }
- }
-
- if !s.Verified {
- if detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
- continue
- }
- }
-
- results = append(results, s)
- }
-
- return
-}
diff --git a/pkg/detectors/midise/midise_test.go b/pkg/detectors/midise/midise_test.go
deleted file mode 100644
index 71c1d23da..000000000
--- a/pkg/detectors/midise/midise_test.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package midise
-
-import (
- "context"
- "testing"
-
- "github.com/kylelemons/godebug/pretty"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-)
-
-func TestTruffle_FromChunk(t *testing.T) {
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- }{
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Midise.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatal("no raw secret present")
- }
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Midise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- for n := 0; n < b.N; n++ {
- _, err := s.FromData(ctx, false, data)
- if err != nil {
- b.Fatal(err)
- }
- }
- })
- }
-}
diff --git a/pkg/detectors/mindmeister/mindmeister.go b/pkg/detectors/mindmeister/mindmeister.go
index 8c27f8025..9503759c1 100644
--- a/pkg/detectors/mindmeister/mindmeister.go
+++ b/pkg/detectors/mindmeister/mindmeister.go
@@ -56,13 +56,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"code":"1010"`)
+ validResponse := strings.Contains(bodyString, `"stat":"ok"`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/mindmeister/mindmeister/mindmeister.go b/pkg/detectors/mindmeister/mindmeister/mindmeister.go
deleted file mode 100644
index 8c27f8025..000000000
--- a/pkg/detectors/mindmeister/mindmeister/mindmeister.go
+++ /dev/null
@@ -1,81 +0,0 @@
-package mindmeister
-
-import (
- "context"
- "fmt"
- "io/ioutil"
- "net/http"
- "regexp"
- "strings"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
- client = common.SaneHttpClient()
-
- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mindmeister"}) + `\b([a-zA-Z0-9]{43})\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
- return []string{"mindmeister"}
-}
-
-// FromData will find and optionally verify Mindmeister secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, match := range matches {
- if len(match) != 2 {
- continue
- }
- resMatch := strings.TrimSpace(match[1])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_Mindmeister,
- Raw: []byte(resMatch),
- }
-
- if verify {
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://www.mindmeister.com/services/rest/oauth2?method=mm.maps.getList", nil)
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
- res, err := client.Do(req)
- if err == nil {
- bodyBytes, err := ioutil.ReadAll(res.Body)
- if err == nil {
- bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"code":"1010"`)
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
- s1.Verified = true
- }
- } else {
- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
- continue
- }
- }
- }
- }
- }
-
- results = append(results, s1)
- }
-
- return detectors.CleanResults(results), nil
-}
diff --git a/pkg/detectors/mindmeister/mindmeister/mindmeister_test.go b/pkg/detectors/mindmeister/mindmeister/mindmeister_test.go
deleted file mode 100644
index 1d4f939f1..000000000
--- a/pkg/detectors/mindmeister/mindmeister/mindmeister_test.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package mindmeister
-
-import (
- "context"
- "fmt"
- "testing"
- "time"
-
- "github.com/kylelemons/godebug/pretty"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestMindmeister_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("MINDMEISTER")
- inactiveSecret := testSecrets.MustGetField("MINDMEISTER_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a mindmeister secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Mindmeister,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a mindmeister secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Mindmeister,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Mindmeister.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Mindmeister.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
- }
- })
- }
-}
diff --git a/pkg/detectors/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock.go
index 0751fbef0..9b7ce5f50 100644
--- a/pkg/detectors/mrticktock/mrticktock.go
+++ b/pkg/detectors/mrticktock/mrticktock.go
@@ -65,7 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Wrong credentials.") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"errors":[]`) {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/mrticktock/mrticktock/mrticktock.go b/pkg/detectors/mrticktock/mrticktock/mrticktock.go
deleted file mode 100644
index 0751fbef0..000000000
--- a/pkg/detectors/mrticktock/mrticktock/mrticktock.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package mrticktock
-
-import (
- "context"
- "fmt"
- "io/ioutil"
- "net/http"
- "regexp"
- "strings"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
- client = common.SaneHttpClient()
-
- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
- emailPat = regexp.MustCompile(`\b([a-zA-Z0-9._-]+@[a-zA-Z0-9._-]+\.[a-z]+)\b`)
- pwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mrticktock"}) + `\b([a-zA-Z0-9!=@#$%()_^]{1,50})`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
- return []string{"mrticktock"}
-}
-
-// FromData will find and optionally verify Mrticktock secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- matches := emailPat.FindAllStringSubmatch(dataStr, -1)
- passwordMatches := pwordPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, match := range matches {
- if len(match) != 2 {
- continue
- }
- resMatch := strings.TrimSpace(match[1])
-
- for _, passwordMatch := range passwordMatches {
- if len(passwordMatch) != 2 {
- continue
- }
- resPassword := strings.TrimSpace(passwordMatch[1])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_Mrticktock,
- Raw: []byte(resMatch),
- }
-
- if verify {
- payload := strings.NewReader(fmt.Sprintf(`email=%s&password=%s`, resMatch, resPassword))
- req, _ := http.NewRequestWithContext(ctx, "POST", "https://mrticktock.com/app/api/is_timer_active", payload)
- req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
- res, err := client.Do(req)
- if err == nil {
- defer res.Body.Close()
- bodyBytes, _ := ioutil.ReadAll(res.Body)
- body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Wrong credentials.") {
- s1.Verified = true
- } else {
- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
- continue
- }
- }
- }
- }
-
- results = append(results, s1)
- }
- }
- return detectors.CleanResults(results), nil
-}
diff --git a/pkg/detectors/mrticktock/mrticktock/mrticktock_test.go b/pkg/detectors/mrticktock/mrticktock/mrticktock_test.go
deleted file mode 100644
index d3a7440ee..000000000
--- a/pkg/detectors/mrticktock/mrticktock/mrticktock_test.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package mrticktock
-
-import (
- "context"
- "fmt"
- "testing"
- "time"
-
- "github.com/kylelemons/godebug/pretty"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestMrticktock_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- email := testSecrets.MustGetField("SCANNERS_EMAIL")
- pword := testSecrets.MustGetField("SCANNERS_PASS")
- inactiveSecret := testSecrets.MustGetField("SCANNERS_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a mrticktock secret %s within %s", email, pword)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Mrticktock,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a mrticktock secret %s within %s but not valid", email, inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_Mrticktock,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("Mrticktock.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("Mrticktock.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
- }
- })
- }
-}
diff --git a/pkg/detectors/onedesk/onedesk.go b/pkg/detectors/onedesk/onedesk.go
index 77e6d0f45..18d9d9ffd 100644
--- a/pkg/detectors/onedesk/onedesk.go
+++ b/pkg/detectors/onedesk/onedesk.go
@@ -64,7 +64,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "ERROR_LOGIN_FAILED") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"code":"SUCCESS"`) {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/paralleldots/paralleldots.go b/pkg/detectors/paralleldots/paralleldots.go
index 3cf960713..a2b5cc27a 100644
--- a/pkg/detectors/paralleldots/paralleldots.go
+++ b/pkg/detectors/paralleldots/paralleldots.go
@@ -77,7 +77,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if (res.StatusCode >= 200 && res.StatusCode < 300) && !strings.Contains(body, "Invalid API Key") {
+ if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "intent") {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/pdflayer/pdflayer.go b/pkg/detectors/pdflayer/pdflayer.go
index 0bf089f0c..0bd108ea4 100644
--- a/pkg/detectors/pdflayer/pdflayer.go
+++ b/pkg/detectors/pdflayer/pdflayer.go
@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"code":101`)
+ validResponse := strings.Contains(bodyString, `Contents`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/scrapersite/scrapersite.go b/pkg/detectors/scrapersite/scrapersite.go
index 804a52bf6..b63abdad4 100644
--- a/pkg/detectors/scrapersite/scrapersite.go
+++ b/pkg/detectors/scrapersite/scrapersite.go
@@ -55,22 +55,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
req, _ := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://scrapersite.com/api-v1?api_key=%s&url=https://google.com", resMatch), nil)
res, err := client.Do(req)
if err == nil {
- bodyBytes, err := ioutil.ReadAll(res.Body)
- if err == nil {
- bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"code":101`)
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
- s1.Verified = true
- }
+ bodyBytes, _ := ioutil.ReadAll(res.Body)
+ bodyString := string(bodyBytes)
+ validResponse := strings.Contains(bodyString, `"status":true`)
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ if validResponse {
+ s1.Verified = true
} else {
- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
- continue
- }
+ s1.Verified = false
+ }
+ } else {
+ //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
}
}
}
diff --git a/pkg/detectors/scrapersite/scrapersite/scrapersite.go b/pkg/detectors/scrapersite/scrapersite/scrapersite.go
deleted file mode 100644
index 632a9597d..000000000
--- a/pkg/detectors/scrapersite/scrapersite/scrapersite.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package scrapersite
-
-import (
- "context"
- "fmt"
- "io/ioutil"
- "net/http"
- "regexp"
- "strings"
- "time"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
- client = common.SaneHttpClient()
-
- //Make sure that your group is surrounded in boundry characters such as below to reduce false positives
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
- return []string{"scrapersite"}
-}
-
-// FromData will find and optionally verify ScraperSite secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, match := range matches {
- if len(match) != 2 {
- continue
- }
- resMatch := strings.TrimSpace(match[1])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_ScraperSite,
- Raw: []byte(resMatch),
- }
-
- if verify {
- timeout := 10 * time.Second
- client.Timeout = timeout
- req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://scrapersite.com/api-v1?api_key=%s&url=https://google.com", resMatch), nil)
- if err != nil {
- continue
- }
- res, err := client.Do(req)
- if err == nil {
- bodyBytes, err := ioutil.ReadAll(res.Body)
- if err == nil {
- bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"code":101`)
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
- s1.Verified = true
- }
- } else {
- //This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
- continue
- }
- }
- }
- }
- }
-
- results = append(results, s1)
- }
-
- return detectors.CleanResults(results), nil
-}
diff --git a/pkg/detectors/scrapersite/scrapersite/scrapersite_test.go b/pkg/detectors/scrapersite/scrapersite/scrapersite_test.go
deleted file mode 100644
index fc0d02a65..000000000
--- a/pkg/detectors/scrapersite/scrapersite/scrapersite_test.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package scrapersite
-
-import (
- "context"
- "fmt"
- "testing"
- "time"
-
- "github.com/kylelemons/godebug/pretty"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestScraperSite_FromChunk(t *testing.T) {
- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
- defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
- if err != nil {
- t.Fatalf("could not get test secrets from GCP: %s", err)
- }
- secret := testSecrets.MustGetField("SCRAPERSITE")
- inactiveSecret := testSecrets.MustGetField("SCRAPERSITE_INACTIVE")
-
- type args struct {
- ctx context.Context
- data []byte
- verify bool
- }
- tests := []struct {
- name string
- s Scanner
- args args
- want []detectors.Result
- wantErr bool
- }{
- {
- name: "found, verified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapersite secret %s within", secret)),
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScraperSite,
- Verified: true,
- },
- },
- wantErr: false,
- },
- {
- name: "found, unverified",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte(fmt.Sprintf("You can find a scrapersite secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
- verify: true,
- },
- want: []detectors.Result{
- {
- DetectorType: detectorspb.DetectorType_ScraperSite,
- Verified: false,
- },
- },
- wantErr: false,
- },
- {
- name: "not found",
- s: Scanner{},
- args: args{
- ctx: context.Background(),
- data: []byte("You cannot find the secret within"),
- verify: true,
- },
- want: nil,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- s := Scanner{}
- got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
- if (err != nil) != tt.wantErr {
- t.Errorf("ScraperSite.FromData() error = %v, wantErr %v", err, tt.wantErr)
- return
- }
- for i := range got {
- if len(got[i].Raw) == 0 {
- t.Fatalf("no raw secret present: \n %+v", got[i])
- }
- got[i].Raw = nil
- }
- if diff := pretty.Compare(got, tt.want); diff != "" {
- t.Errorf("ScraperSite.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
- }
- })
- }
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
- ctx := context.Background()
- s := Scanner{}
- for name, data := range detectors.MustGetBenchmarkData() {
- benchmark.Run(name, func(b *testing.B) {
- for n := 0; n < b.N; n++ {
- s.FromData(ctx, false, data)
- }
- })
- }
-}
diff --git a/pkg/detectors/semaphore/semaphore.go b/pkg/detectors/semaphore/semaphore.go
index d7c2669a7..2b1fd099c 100644
--- a/pkg/detectors/semaphore/semaphore.go
+++ b/pkg/detectors/semaphore/semaphore.go
@@ -60,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "apikey is invalid") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, "account_id") {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/upcdatabase/upcdatabase.go b/pkg/detectors/upcdatabase/upcdatabase.go
index 2da90b80a..825191bbb 100644
--- a/pkg/detectors/upcdatabase/upcdatabase.go
+++ b/pkg/detectors/upcdatabase/upcdatabase.go
@@ -55,13 +55,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"error":{"message":"Your API Key is invalid.`)
+ validResponse := strings.Contains(bodyString, `added_time`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/walkscore/walkscore.go b/pkg/detectors/walkscore/walkscore.go
index 75ee5df26..e3c5301a3 100644
--- a/pkg/detectors/walkscore/walkscore.go
+++ b/pkg/detectors/walkscore/walkscore.go
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && !strings.Contains(body, `Invalid 'wsapikey'`)) {
+ if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, `distance`) {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/webscraping/webscraping.go b/pkg/detectors/webscraping/webscraping.go
index 80da03b3a..1836d455d 100644
--- a/pkg/detectors/webscraping/webscraping.go
+++ b/pkg/detectors/webscraping/webscraping.go
@@ -48,14 +48,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://api.webscrapingapi.com/v1?api_key="+resMatch, nil)
+ req, _ := http.NewRequestWithContext(ctx, "GET", `https://api.webscrapingapi.com/v1?api_key=`+resMatch, nil)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if !strings.Contains(body, "Invalid API key") {
+ if strings.Contains(body, "key `url` required.") {
s1.Verified = true
} else {
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
diff --git a/pkg/detectors/whoxy/whoxy.go b/pkg/detectors/whoxy/whoxy.go
index 6e01c8bc8..863340a65 100644
--- a/pkg/detectors/whoxy/whoxy.go
+++ b/pkg/detectors/whoxy/whoxy.go
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
defer res.Body.Close()
bodyBytes, _ := ioutil.ReadAll(res.Body)
body := string(bodyBytes)
- if res.StatusCode >= 200 && res.StatusCode < 300 && !strings.Contains(body, "Incorrect API Key") {
+ if res.StatusCode >= 200 && res.StatusCode < 300 && strings.Contains(body, `"status": 1`) {
s1.Verified = true
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/worldcoinindex/worldcoinindex.go b/pkg/detectors/worldcoinindex/worldcoinindex.go
index 782ffc768..cf6f32d9f 100644
--- a/pkg/detectors/worldcoinindex/worldcoinindex.go
+++ b/pkg/detectors/worldcoinindex/worldcoinindex.go
@@ -55,14 +55,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
bodyBytes, err := ioutil.ReadAll(res.Body)
if err == nil {
bodyString := string(bodyBytes)
- errCode := strings.Contains(bodyString, `"error"`)
+ validResponse := strings.Contains(bodyString, `"Markets"`)
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
- if errCode {
- s1.Verified = false
- } else {
+ if validResponse {
s1.Verified = true
+ } else {
+ s1.Verified = false
}
} else {
//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
diff --git a/pkg/detectors/zenscrape/zenscrape.go b/pkg/detectors/zenscrape/zenscrape.go
index 29fae3eed..9201dc207 100644
--- a/pkg/detectors/zenscrape/zenscrape.go
+++ b/pkg/detectors/zenscrape/zenscrape.go
@@ -49,7 +49,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- req, _ := http.NewRequestWithContext(ctx, "GET", "https://app.zenscrape.com/api/v1/get", nil)
+ req, _ := http.NewRequestWithContext(ctx, "GET", "https://app.zenscrape.com/api/v1/get?url=http://www.google.com", nil)
req.Header.Add("apikey", fmt.Sprintf("%s", resMatch))
res, err := client.Do(req)
if err == nil {