Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
feat(openai): add project and service account keys (#2863)
Richard Gomez024b219dfbb1
diff --git a/pkg/detectors/openai/openai.go b/pkg/detectors/openai/openai.goindex 5e4442786..1b0e53ca3 100644--- a/pkg/detectors/openai/openai.go+++ b/pkg/detectors/openai/openai.go@@ -4,38 +4,31 @@ import ("context""encoding/json""fmt"- regexp "github.com/wasilibs/go-re2"+ "io""net/http""strconv"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb""github.com/trufflesecurity/trufflehog/v3/pkg/common")-type Scanner struct{}+type Scanner struct {+ client *http.Client+}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)-// The magic string T3BlbkFJ is the base64-encoded string: OpenAI-var keyPat = regexp.MustCompile(`\b(sk-[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20})\b`)+var (+ defaultClient = common.SaneHttpClient()-// TODO: Add secret context?? Information about access, ownership etc-type orgResponse struct {- Data []organization `json:"data"`-}--type organization struct {- ID string `json:"id"`- Title string `json:"title"`- User string `json:"name"`- Description string `json:"description"`- Personal bool `json:"personal"`- Default bool `json:"is_default"`- Role string `json:"role"`-}+ // The magic string T3BlbkFJ is the base64-encoded string: OpenAI+ keyPat = regexp.MustCompile(`\b(sk-(?:(?:proj|[a-z0-9](?:[a-z0-9-]{0,40}[a-z0-9])?)-)?[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20})\b`)+)// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.@@ -47,16 +40,12 @@ func (s Scanner) Keywords() []string {func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllStringSubmatch(dataStr, -1)-- for _, match := range matches {- // First match is entire regex, second is the first group.- if len(match) != 2 {- continue- }-- token := match[1]+ uniqueMatches := make(map[string]struct{})+ for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueMatches[match[1]] = struct{}{}+ }+ for token := range uniqueMatches {s1 := detectors.Result{DetectorType: detectorspb.DetectorType_OpenAI,Redacted: token[:3] + "..." + token[47:],@@ -64,37 +53,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- client := common.SaneHttpClient()- // Undocumented API- // https://api.openai.com/v1/organizations- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)- if err != nil {- continue- }- req.Header.Add("Content-Type", "application/json; charset=utf-8")- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))- res, err := client.Do(req)- if err == nil {- if res.StatusCode >= 200 && res.StatusCode < 300 {- var orgs orgResponse- err = json.NewDecoder(res.Body).Decode(&orgs)- res.Body.Close()- if err == nil {- s1.Verified = true- org := orgs.Data[0]- s1.ExtraData = map[string]string{- "id": org.ID,- "title": org.Title,- "user": org.User,- "description": org.Description,- "role": org.Role,- "is_personal": strconv.FormatBool(org.Personal),- "is_default": strconv.FormatBool(org.Default),- "total_orgs": fmt.Sprintf("%d", len(orgs.Data)),- }- }- }+ client := s.client+ if client == nil {+ client = defaultClient}++ verified, extraData, verificationErr := verifyToken(ctx, client, token)+ s1.Verified = verified+ s1.ExtraData = extraData+ s1.SetVerificationError(verificationErr)}results = append(results, s1)@@ -103,6 +70,67 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn}+func verifyToken(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {+ // Undocumented API+ // https://api.openai.com/v1/organizations+ req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)+ if err != nil {+ return false, nil, err+ }++ req.Header.Set("Content-Type", "application/json; charset=utf-8")+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))+ res, err := client.Do(req)+ if err != nil {+ return false, nil, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()++ switch res.StatusCode {+ case 200:+ var orgs orgResponse+ if err = json.NewDecoder(res.Body).Decode(&orgs); err != nil {+ return false, nil, err+ }++ org := orgs.Data[0]+ extraData := map[string]string{+ "id": org.ID,+ "title": org.Title,+ "user": org.User,+ "description": org.Description,+ "role": org.Role,+ "is_personal": strconv.FormatBool(org.Personal),+ "is_default": strconv.FormatBool(org.Default),+ "total_orgs": fmt.Sprintf("%d", len(orgs.Data)),+ }+ return true, extraData, nil+ case 401:+ // Invalid+ return false, nil, nil+ default:+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ }+}++// TODO: Add secret context?? Information about access, ownership etc+type orgResponse struct {+ Data []organization `json:"data"`+}++type organization struct {+ ID string `json:"id"`+ Title string `json:"title"`+ User string `json:"name"`+ Description string `json:"description"`+ Personal bool `json:"personal"`+ Default bool `json:"is_default"`+ Role string `json:"role"`+}+func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_OpenAI}diff --git a/pkg/detectors/openai/openai_test.go b/pkg/detectors/openai/openai_test.goindex 2fda86e46..6035aa206 100644--- a/pkg/detectors/openai/openai_test.go+++ b/pkg/detectors/openai/openai_test.go@@ -9,12 +9,84 @@ import ("testing""time"+ "github.com/google/go-cmp/cmp""github.com/kylelemons/godebug/pretty"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")+func TestOpenAI_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "user API key",+ input: "openai.api-key: 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",+ want: []string{"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"},+ },+ {+ name: "project API key",+ input: `OPENAI_API_KEY = "sk-proj-mpjtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiagE8K"`,+ want: []string{"sk-proj-mpjtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiagE8K"},+ },+ {+ name: "service account API key",+ input: `OPENAI_API_KEY = "sk-service-account-name-Ofbtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiaglyC"`,+ want: []string{"sk-service-account-name-Ofbtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiaglyC"},+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2)+ ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors)+ if len(chunkSpecificDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}+func TestOpenAI_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()