Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Log recursion limiting (#4236)
Charlie Gunyonead4289a6dc9
diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.goindex 6d96c0a0b..db8e81b26 100644--- a/pkg/sources/postman/postman.go+++ b/pkg/sources/postman/postman.go@@ -329,10 +329,13 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu// scanCollection scans a collection and all its items, folders, and requests.// locally scoped Metadata is updated as we drill down into the collection.func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, collection Collection) {- ctx.Logger().V(2).Info("starting to scan collection",+ ctx = context.WithValues(ctx,"collection_name", collection.Info.Name,"collection_uuid", collection.Info.Uid,- "variable_count", len(collection.Variables))+ )+ ctx.Logger().V(2).Info("starting to scan collection",+ "variable_count", len(collection.Variables),+ )metadata.CollectionInfo = collection.Infometadata.Type = COLLECTION_TYPEs.attemptToAddKeyword(collection.Info.Name)@@ -354,7 +357,6 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Chs.scanAuth(ctx, chunksChan, metadata, collection.Auth, URL{})ctx.Logger().V(3).Info("Scanning events in collection",- "collection_uid", collection.Info.Uid,"event_count", len(collection.Events),)for _, event := range collection.Events {@@ -362,24 +364,27 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch}ctx.Logger().V(3).Info("Scanning items in collection",- "collection_uid", collection.Info.Uid,"item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items),)for _, item := range collection.Items {- s.scanItem(ctx, chunksChan, collection, metadata, item, "")+ seenItemIds := make(map[string]struct{})+ s.scanItem(ctx, chunksChan, collection, metadata, item, "", seenItemIds)}-}-func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string) {+func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string, seenItemIds map[string]struct{}) {+ ctx = context.WithValue(ctx, "item_uid", item.Uid)+ctx.Logger().V(3).Info("Starting to scan item",- "item_uid", item.Uid,"item_parent_item_id", parentItemId,"item_descendent_item_uids", fp.Map(func(i Item) string { return i.Uid })(item.Items),"item_event_count", len(item.Events),"item_response_count", len(item.Response),"item_variable_count", len(item.Variable),)++ seenItemIds[item.Uid] = struct{}{}+s.attemptToAddKeyword(item.Name)// override the base collection metadata with item-specific metadata@@ -397,7 +402,13 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c}// recurse through the foldersfor _, subItem := range item.Items {- s.scanItem(ctx, chunksChan, collection, metadata, subItem, item.Uid)+ if _, ok := seenItemIds[subItem.Uid]; ok {+ ctx.Logger().Info("Skipping already-seen item",+ "seen_item_id", subItem.Uid,+ )+ continue+ }+ s.scanItem(ctx, chunksChan, collection, metadata, subItem, item.Uid, seenItemIds)}// The assignment of the folder ID to be the current item UID is due to wanting to assume that your current item is a folder unless you have request data inside of your item.@@ -463,7 +474,8 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT}- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, data, DefaultMaxRecursionDepth)), metadata)+ ctx = context.WithValue(ctx, "event_listen", event.Listen)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, data, DefaultMaxRecursionDepth)), metadata)metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -561,7 +573,7 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m} else if strings.Contains(m.Type, COLLECTION_TYPE) {m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION}- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, authData, DefaultMaxRecursionDepth)), m)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, authData, DefaultMaxRecursionDepth)), m)m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -591,7 +603,7 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Cmetadata.Type = originalType + " > header"metadata.Link = metadata.Link + "?tab=headers"metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, strings.Join(r.HeaderString, " "), DefaultMaxRecursionDepth)), metadata)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, strings.Join(r.HeaderString, " "), DefaultMaxRecursionDepth)), metadata)metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -600,7 +612,7 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C// Note: query parameters are handled separatelyu := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(metadata, u, DefaultMaxRecursionDepth)), metadata)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, u, DefaultMaxRecursionDepth)), metadata)metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -655,13 +667,13 @@ func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.Cm.Type = originalType + " > raw"data := b.Rawm.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_RAW- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data, DefaultMaxRecursionDepth)), m)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, data, DefaultMaxRecursionDepth)), m)m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMANcase "graphql":m.Type = originalType + " > graphql"data := b.GraphQL.Query + " " + b.GraphQL.Variablesm.LocationType = source_metadatapb.PostmanLocationType_REQUEST_BODY_GRAPHQL- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, data, DefaultMaxRecursionDepth)), m)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, data, DefaultMaxRecursionDepth)), m)m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}}@@ -687,7 +699,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.m.Type = originalType + " > response header"// TODO Note: for now, links to Postman responses do not include a more granular tab for the params/header/body, but when they do, we will need to update the metadata.Link infom.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, strings.Join(response.HeaderString, " "), DefaultMaxRecursionDepth)), m)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, strings.Join(response.HeaderString, " "), DefaultMaxRecursionDepth)), m)m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -695,7 +707,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.if response.Body != "" {m.Type = originalType + " > response body"m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY- s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(m, response.Body, DefaultMaxRecursionDepth)), m)+ s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, m, response.Body, DefaultMaxRecursionDepth)), m)m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -728,7 +740,7 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.if valStr == "" {continue}- values = append(values, s.buildSubstituteSet(m, valStr, DefaultMaxRecursionDepth)...)+ values = append(values, s.buildSubstituteSet(ctx, m, valStr, DefaultMaxRecursionDepth)...)}m.FieldType = m.Type + " variables"diff --git a/pkg/sources/postman/substitution.go b/pkg/sources/postman/substitution.goindex 2b5b3ecbc..9f68f4f6e 100644--- a/pkg/sources/postman/substitution.go+++ b/pkg/sources/postman/substitution.go@@ -4,6 +4,8 @@ import ("fmt""regexp""strings"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context")var subRe = regexp.MustCompile(`\{\{[^{}]+\}\}`)@@ -52,12 +54,17 @@ func (s *Source) formatAndInjectKeywords(data []string) string {// buildSubstituteSet creates a set of substitutions for the given data// maxRecursionDepth is the maximum recursion depth to use for variable substitution-func (s *Source) buildSubstituteSet(metadata Metadata, data string, maxRecursionDepth int) []string {+func (s *Source) buildSubstituteSet(+ ctx context.Context,+ metadata Metadata,+ data string,+ maxRecursionDepth int,+) []string {var ret []stringcombos := make(map[string]struct{})// Call buildSubstitution with initial depth of 0 and the maxRecursionDepth- s.buildSubstitution(data, metadata, &combos, 0, maxRecursionDepth)+ s.buildSubstitution(ctx, data, metadata, combos, 0, maxRecursionDepth)for combo := range combos {ret = append(ret, combo)@@ -73,22 +80,26 @@ func (s *Source) buildSubstituteSet(metadata Metadata, data string, maxRecursion// depth is the current recursion depth// maxRecursionDepth is the maximum recursion depth to use for variable substitutionfunc (s *Source) buildSubstitution(+ ctx context.Context,data string,metadata Metadata,- combos *map[string]struct{},+ combos map[string]struct{},depth int,maxRecursionDepth int,) {// Limit recursion depth to prevent stack overflowif depth > maxRecursionDepth {- (*combos)[data] = struct{}{}+ ctx.Logger().V(2).Info("Limited recursion depth",+ "depth", depth,+ )+ combos[data] = struct{}{}return}matches := removeDuplicateStr(subRe.FindAllString(data, -1))if len(matches) == 0 {// No more substitutions to make, add to combos- (*combos)[data] = struct{}{}+ combos[data] = struct{}{}return}@@ -117,14 +128,14 @@ func (s *Source) buildSubstitution(// Only mark substitution as made if we actually changed somethingif d != data {substitutionMade = true- s.buildSubstitution(d, metadata, combos, depth+1, maxRecursionDepth)+ s.buildSubstitution(ctx, d, metadata, combos, depth+1, maxRecursionDepth)}}}// If no substitutions were made, add the current dataif !substitutionMade {- (*combos)[data] = struct{}{}+ combos[data] = struct{}{}}}diff --git a/pkg/sources/postman/substitution_test.go b/pkg/sources/postman/substitution_test.goindex 5204ccb1a..e5ba5e44c 100644--- a/pkg/sources/postman/substitution_test.go+++ b/pkg/sources/postman/substitution_test.go@@ -5,6 +5,8 @@ import ("sort""strings""testing"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context")func TestNewSubstitution(t *testing.T) {@@ -63,6 +65,8 @@ func TestSource_KeywordCombinations(t *testing.T) {}func TestSource_BuildSubstituteSet(t *testing.T) {+ ctx := context.Background()+s := &Source{sub: NewSubstitution(),}@@ -89,7 +93,7 @@ func TestSource_BuildSubstituteSet(t *testing.T) {}for _, tc := range testCases {- result := s.buildSubstituteSet(metadata, tc.data, DefaultMaxRecursionDepth)+ result := s.buildSubstituteSet(ctx, metadata, tc.data, DefaultMaxRecursionDepth)if !reflect.DeepEqual(result, tc.expected) {t.Errorf("Expected substitution set: %v, got: %v", tc.expected, result)}@@ -158,6 +162,8 @@ func TestSource_FormatAndInjectKeywords(t *testing.T) {}func TestSource_BuildSubstitution_RecursionLimit(t *testing.T) {+ ctx := context.Background()+s := &Source{sub: NewSubstitution(),}@@ -229,9 +235,9 @@ func TestSource_BuildSubstitution_RecursionLimit(t *testing.T) {// Use custom maxDepth if provided, otherwise use defaultif tc.maxDepth > 0 {- s.buildSubstitution(tc.data, metadata, &combos, 0, tc.maxDepth)+ s.buildSubstitution(ctx, tc.data, metadata, combos, 0, tc.maxDepth)} else {- s.buildSubstitution(tc.data, metadata, &combos, 0, DefaultMaxRecursionDepth)+ s.buildSubstitution(ctx, tc.data, metadata, combos, 0, DefaultMaxRecursionDepth)}var result []string