Skip to content

Find, verify, and analyze leaked credentials

feat(openai): add project and service account keys (#2863)

Richard Gomez024b219dfbb1
diff --git a/pkg/detectors/openai/openai.go b/pkg/detectors/openai/openai.go
index 5e4442786..1b0e53ca3 100644
--- a/pkg/detectors/openai/openai.go
+++ b/pkg/detectors/openai/openai.go
@@ -4,38 +4,31 @@ import (
"context"
"encoding/json"
"fmt"
- regexp "github.com/wasilibs/go-re2"
+ "io"
"net/http"
"strconv"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
-type Scanner struct{}
+type Scanner struct {
+ client *http.Client
+}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
-// The magic string T3BlbkFJ is the base64-encoded string: OpenAI
-var keyPat = regexp.MustCompile(`\b(sk-[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20})\b`)
+var (
+ defaultClient = common.SaneHttpClient()
-// TODO: Add secret context?? Information about access, ownership etc
-type orgResponse struct {
- Data []organization `json:"data"`
-}
-
-type organization struct {
- ID string `json:"id"`
- Title string `json:"title"`
- User string `json:"name"`
- Description string `json:"description"`
- Personal bool `json:"personal"`
- Default bool `json:"is_default"`
- Role string `json:"role"`
-}
+ // The magic string T3BlbkFJ is the base64-encoded string: OpenAI
+ keyPat = regexp.MustCompile(`\b(sk-(?:(?:proj|[a-z0-9](?:[a-z0-9-]{0,40}[a-z0-9])?)-)?[[:alnum:]]{20}T3BlbkFJ[[:alnum:]]{20})\b`)
+)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
@@ -47,16 +40,12 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, match := range matches {
- // First match is entire regex, second is the first group.
- if len(match) != 2 {
- continue
- }
-
- token := match[1]
+ uniqueMatches := make(map[string]struct{})
+ for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
+ uniqueMatches[match[1]] = struct{}{}
+ }
+ for token := range uniqueMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_OpenAI,
Redacted: token[:3] + "..." + token[47:],
@@ -64,37 +53,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- client := common.SaneHttpClient()
- // Undocumented API
- // https://api.openai.com/v1/organizations
- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)
- if err != nil {
- continue
- }
- req.Header.Add("Content-Type", "application/json; charset=utf-8")
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
- res, err := client.Do(req)
- if err == nil {
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- var orgs orgResponse
- err = json.NewDecoder(res.Body).Decode(&orgs)
- res.Body.Close()
- if err == nil {
- s1.Verified = true
- org := orgs.Data[0]
- s1.ExtraData = map[string]string{
- "id": org.ID,
- "title": org.Title,
- "user": org.User,
- "description": org.Description,
- "role": org.Role,
- "is_personal": strconv.FormatBool(org.Personal),
- "is_default": strconv.FormatBool(org.Default),
- "total_orgs": fmt.Sprintf("%d", len(orgs.Data)),
- }
- }
- }
+ client := s.client
+ if client == nil {
+ client = defaultClient
}
+
+ verified, extraData, verificationErr := verifyToken(ctx, client, token)
+ s1.Verified = verified
+ s1.ExtraData = extraData
+ s1.SetVerificationError(verificationErr)
}
results = append(results, s1)
@@ -103,6 +70,67 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return
}
+func verifyToken(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
+ // Undocumented API
+ // https://api.openai.com/v1/organizations
+ req, err := http.NewRequestWithContext(ctx, "GET", "https://api.openai.com/v1/organizations", nil)
+ if err != nil {
+ return false, nil, err
+ }
+
+ req.Header.Set("Content-Type", "application/json; charset=utf-8")
+ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
+ res, err := client.Do(req)
+ if err != nil {
+ return false, nil, err
+ }
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
+
+ switch res.StatusCode {
+ case 200:
+ var orgs orgResponse
+ if err = json.NewDecoder(res.Body).Decode(&orgs); err != nil {
+ return false, nil, err
+ }
+
+ org := orgs.Data[0]
+ extraData := map[string]string{
+ "id": org.ID,
+ "title": org.Title,
+ "user": org.User,
+ "description": org.Description,
+ "role": org.Role,
+ "is_personal": strconv.FormatBool(org.Personal),
+ "is_default": strconv.FormatBool(org.Default),
+ "total_orgs": fmt.Sprintf("%d", len(orgs.Data)),
+ }
+ return true, extraData, nil
+ case 401:
+ // Invalid
+ return false, nil, nil
+ default:
+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
+ }
+}
+
+// TODO: Add secret context?? Information about access, ownership etc
+type orgResponse struct {
+ Data []organization `json:"data"`
+}
+
+type organization struct {
+ ID string `json:"id"`
+ Title string `json:"title"`
+ User string `json:"name"`
+ Description string `json:"description"`
+ Personal bool `json:"personal"`
+ Default bool `json:"is_default"`
+ Role string `json:"role"`
+}
+
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_OpenAI
}
diff --git a/pkg/detectors/openai/openai_test.go b/pkg/detectors/openai/openai_test.go
index 2fda86e46..6035aa206 100644
--- a/pkg/detectors/openai/openai_test.go
+++ b/pkg/detectors/openai/openai_test.go
@@ -9,12 +9,84 @@ import (
"testing"
"time"
+ "github.com/google/go-cmp/cmp"
"github.com/kylelemons/godebug/pretty"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
+func TestOpenAI_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+ tests := []struct {
+ name string
+ input string
+ want []string
+ }{
+ {
+ name: "user API key",
+ input: "openai.api-key: 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",
+ want: []string{"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"},
+ },
+ {
+ name: "project API key",
+ input: `OPENAI_API_KEY = "sk-proj-mpjtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiagE8K"`,
+ want: []string{"sk-proj-mpjtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiagE8K"},
+ },
+ {
+ name: "service account API key",
+ input: `OPENAI_API_KEY = "sk-service-account-name-Ofbtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiaglyC"`,
+ want: []string{"sk-service-account-name-Ofbtr05CFsJqs4TAeKlCT3BlbkFJsh1KtN0SUjTPeJiaglyC"},
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2)
+ ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors)
+ if len(chunkSpecificDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
+ })
+ }
+}
+
func TestOpenAI_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()