Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Clean up Postman source logging for per-source debug (#5510)
Kashif Khan98e399ef21e8
diff --git a/pkg/sources/postman/README.md b/pkg/sources/postman/README.mdindex d8df89122..bd1db403e 100644--- a/pkg/sources/postman/README.md+++ b/pkg/sources/postman/README.md@@ -222,6 +222,18 @@ trufflehog postman --token PMAK-xxxxxxxxxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxtrufflehog postman --collection-paths /path/to/collection.json --environment-paths /path/to/environment.json```+## Debug Logs++With debug on for this source (`debug: true` in the config), the scan logs:++- `V(1)`: `Postman source initialized` with the config choices (the credential type only, never the token), the start and end of workspace enumeration, and `Monthly API limit threshold reached, aborting scan` when the monthly API limit stops a scan.+- `V(2)`: `Scanning workspace`, `Scanning collection` and `Scanning local file`, with their counts.+- `V(3)`: `Scanning environment` and `Scanning item` for each environment and each folder or request.+- `V(4)`: `Skipping collection`, `Skipping environment`, `Skipping item` and `Skipping basic auth` with a `reason`, `Postman API request failed` for any API response of 400 or above (with the status, path and `Retry-After` header), `Retrying Postman API request` for each retry, and `Limited recursion depth` when variable substitution stops early.+- `V(5)`: each API request and response status, the IDs inside a workspace or collection, request details, and `Sending chunk` for each chunk sent to the detectors.++Lines carry `workspace_id`, `collection_uuid`, `item_uid` and `event_listen` where they apply. Data is only sent to the detectors when a detector keyword is found nearby, so a field with no keyword logs `Data string is empty` at `V(5)`.+## Troubleshooting### Common Issuesdiff --git a/pkg/sources/postman/apilog.go b/pkg/sources/postman/apilog.gonew file mode 100644index 000000000..06eab9fdb--- /dev/null+++ b/pkg/sources/postman/apilog.go@@ -0,0 +1,46 @@+package postman++import (+ "net/http"++ "github.com/hashicorp/go-retryablehttp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++// withAPILogHooks logs retries and failed responses from the Postman API. The client+// retries 429 and 5xx on its own, so without these hooks support can't see them. The+// hooks log through the request's context, so every request must carry the scan ctx.+func withAPILogHooks() common.ClientOption {+ return func(c *retryablehttp.Client) {+ c.RequestLogHook = logAPIRetry+ c.ResponseLogHook = logAPIResponse+ }+}++// logAPIRetry is a retryablehttp.RequestLogHook. It runs before every attempt and+// logs the ones that are retries.+func logAPIRetry(_ retryablehttp.Logger, req *http.Request, attempt int) {+ if attempt == 0 {+ return+ }+ trContext.AddLogger(req.Context()).Logger().V(4).Info("Retrying Postman API request",+ "path", req.URL.Path,+ "attempt", attempt,+ )+}++// logAPIResponse is a retryablehttp.ResponseLogHook. It logs failed responses,+// including the ones that will be retried. Headers are not logged, they can hold+// cookies or tokens.+func logAPIResponse(_ retryablehttp.Logger, resp *http.Response) {+ if resp.StatusCode < 400 {+ return+ }+ trContext.AddLogger(resp.Request.Context()).Logger().V(4).Info("Postman API request failed",+ "path", resp.Request.URL.Path,+ "status", resp.StatusCode,+ "retry_after", resp.Header.Get("Retry-After"),+ )+}diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.goindex c0a7fccc1..95908e5e0 100644--- a/pkg/sources/postman/postman.go+++ b/pkg/sources/postman/postman.go@@ -128,9 +128,34 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, soureturn errors.New("credential type not implemented for Postman")}+ ctx.Logger().V(1).Info("Postman source initialized",+ "auth_method", authMethod(&conn),+ "workspaces", len(conn.Workspaces),+ "collections", len(conn.Collections),+ "environments", len(conn.Environments),+ "include_collections", len(conn.IncludeCollections),+ "exclude_collections", len(conn.ExcludeCollections),+ "include_environments", len(conn.IncludeEnvironments),+ "exclude_environments", len(conn.ExcludeEnvironments),+ "local_workspaces", len(conn.WorkspacePaths),+ "local_collections", len(conn.CollectionPaths),+ "local_environments", len(conn.EnvironmentPaths),+ )return nil}+// authMethod names the credential type for logs. It never returns the credential.+func authMethod(conn *sourcespb.Postman) string {+ switch conn.Credential.(type) {+ case *sourcespb.Postman_Token:+ return "token"+ case *sourcespb.Postman_Unauthenticated:+ return "unauthenticated"+ default:+ return "unknown"+ }+}+// Chunks scans the Postman source and sends the data to the chunks chan.// It scans the local environment, collection, and workspace files, and then scans the Postman API if a token is provided.// The Postman source is different to our other sources in that we are not relying on the data we read from the source to contain@@ -142,6 +167,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .// Scan local environmentsfor _, envPath := range s.conn.EnvironmentPaths {+ ctx.Logger().V(2).Info("Scanning local file", "kind", "environment", "path", envPath)env := VariableData{}contents, err := os.ReadFile(envPath)if err != nil {@@ -155,6 +181,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .// Scan local collectionsfor _, collectionPath := range s.conn.CollectionPaths {+ ctx.Logger().V(2).Info("Scanning local file", "kind", "collection", "path", collectionPath)collection := Collection{}contents, err := os.ReadFile(collectionPath)if err != nil {@@ -168,6 +195,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .// Scan local workspacesfor _, workspacePath := range s.conn.WorkspacePaths {+ ctx.Logger().V(2).Info("Scanning local file", "kind", "workspace", "path", workspacePath)// check if zip fileworkspace := Workspace{}if strings.HasSuffix(workspacePath, ".zip") {@@ -191,11 +219,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .}// Log and move on, because sometimes the Postman API seems to give us workspace IDs// that we don't have access to, so we don't want to kill the scan because of it.- ctx.Logger().Error(err, "error getting workspace %s", workspaceID)+ ctx.Logger().Error(err, "Error getting workspace", "workspace_id", workspaceID)continue}s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspaceID), "")- ctx.Logger().V(2).Info("scanning workspace from workspaces given", "workspace", workspaceID)if err = s.scanWorkspace(ctx, chunksChan, w); err != nil {return fmt.Errorf("error scanning workspace %s: %w", workspaceID, err)}@@ -204,6 +231,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .// Scan collectionsfor _, collectionID := range s.conn.Collections {if shouldSkip(collectionID, s.conn.IncludeCollections, s.conn.ExcludeCollections) {+ ctx.Logger().V(4).Info("Skipping collection", "collection_uuid", collectionID, "reason", "matches filter")continue}@@ -214,7 +242,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .}// Log and move on, because sometimes the Postman API seems to give us collection IDs// that we don't have access to, so we don't want to kill the scan because of it.- ctx.Logger().Error(err, "error getting collection %s", collectionID)+ ctx.Logger().Error(err, "Error getting collection", "collection_uuid", collectionID)continue}s.SetProgressOngoing(fmt.Sprintf("Scanning collection %s", collectionID), "")@@ -227,7 +255,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .if err != nil {return fmt.Errorf("error enumerating postman workspaces: %w", err)}- ctx.Logger().V(2).Info("enumerated workspaces", "workspaces", workspaces)+ ctx.Logger().V(1).Info("Enumerated workspaces", "count", len(workspaces))for _, workspace := range workspaces {s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspace.Id), "")if err = s.scanWorkspace(ctx, chunksChan, workspace); err != nil {@@ -262,13 +290,19 @@ func (s *Source) scanLocalWorkspace(ctx context.Context, chunksChan chan *source}func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chunk, workspace Workspace) error {- ctx.Logger().V(4).Info("scanning workspace",- "workspace_id", workspace.Id,- "collection_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Collections),- "environment_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Environments),- "collection_raw_uids", fp.Map(func(c Collection) string { return c.Info.Uid })(workspace.CollectionsRaw),- "environment_raw_ids", fp.Map(func(v VariableData) string { return v.Id })(workspace.EnvironmentsRaw),+ ctx = context.WithValues(ctx, "workspace_id", workspace.Id)+ ctx.Logger().V(2).Info("Scanning workspace",+ "collection_count", len(workspace.Collections),+ "environment_count", len(workspace.Environments),)+ if ctx.Logger().V(5).Enabled() {+ ctx.Logger().V(5).Info("Workspace contents",+ "collection_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Collections),+ "environment_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Environments),+ "collection_raw_uids", fp.Map(func(c Collection) string { return c.Info.Uid })(workspace.CollectionsRaw),+ "environment_raw_ids", fp.Map(func(v VariableData) string { return v.Id })(workspace.EnvironmentsRaw),+ )+ }// reset keywords for each workspaces.resetKeywords()s.attemptToAddKeyword(workspace.Name)@@ -283,6 +317,10 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu// gather and scan environment variablesfor _, envID := range workspace.Environments {+ if shouldSkip(envID.Uid, s.conn.IncludeEnvironments, s.conn.ExcludeEnvironments) {+ ctx.Logger().V(4).Info("Skipping environment", "environment_uuid", envID.Uid, "reason", "matches filter")+ continue+ }envVars, err := s.client.GetEnvironmentVariables(ctx, envID.Uid)if err != nil {if errors.Is(err, errAbortScanDueToAPIRateLimit) {@@ -291,16 +329,13 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chuctx.Logger().Error(err, "could not get env variables", "environment_uuid", envID.Uid)continue}- if shouldSkip(envID.Uid, s.conn.IncludeEnvironments, s.conn.ExcludeEnvironments) {- continue- }metadata.Type = ENVIRONMENT_TYPEmetadata.Link = LINK_BASE_URL + "environments/" + envID.Uidmetadata.FullID = envVars.Idmetadata.EnvironmentID = envID.Uidmetadata.EnvironmentName = envVars.Name- ctx.Logger().V(2).Info("scanning environment vars", "environment_uuid", metadata.FullID)+ ctx.Logger().V(3).Info("Scanning environment", "environment_uuid", envID.Uid, "variable_count", len(envVars.KeyValues))for _, word := range strings.Split(envVars.Name, " ") {s.attemptToAddKeyword(word)}@@ -312,15 +347,14 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chumetadata.FullID = ""metadata.EnvironmentID = ""metadata.EnvironmentName = ""- ctx.Logger().V(2).Info("finished scanning environment vars", "environment_uuid", metadata.FullID)}- ctx.Logger().V(2).Info("finished scanning environments")// scan all the collections in the workspace.// at this point we have all the possible// substitutions from Environment variablesfor _, collectionID := range workspace.Collections {if shouldSkip(collectionID.Uid, s.conn.IncludeCollections, s.conn.ExcludeCollections) {+ ctx.Logger().V(4).Info("Skipping collection", "collection_uuid", collectionID.Uid, "reason", "matches filter")continue}collection, err := s.client.GetCollection(ctx, collectionID.Uid)@@ -330,7 +364,7 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu}// Log and move on, because sometimes the Postman API seems to give us collection IDs// that we don't have access to, so we don't want to kill the scan because of it.- ctx.Logger().Error(err, "error getting collection %s", collectionID)+ ctx.Logger().Error(err, "Error getting collection", "collection_uuid", collectionID.Uid)continue}s.scanCollection(ctx, chunksChan, metadata, collection)@@ -345,9 +379,16 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch"collection_name", collection.Info.Name,"collection_uuid", collection.Info.Uid,)- ctx.Logger().V(2).Info("starting to scan collection",+ ctx.Logger().V(2).Info("Scanning collection","variable_count", len(collection.Variables),+ "event_count", len(collection.Events),+ "item_count", len(collection.Items),)+ if ctx.Logger().V(5).Enabled() {+ ctx.Logger().V(5).Info("Collection items",+ "item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items),+ )+ }metadata.CollectionInfo = collection.Infometadata.Type = COLLECTION_TYPEs.attemptToAddKeyword(collection.Info.Name)@@ -368,16 +409,10 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch// collections don't have URLs in the Postman API, but we can scan the Authorization section without it.s.scanAuth(ctx, chunksChan, metadata, collection.Auth, URL{})- ctx.Logger().V(3).Info("Scanning events in collection",- "event_count", len(collection.Events),- )for _, event := range collection.Events {s.scanEvent(ctx, chunksChan, metadata, event)}- ctx.Logger().V(3).Info("Scanning items in collection",- "item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items),- )for _, item := range collection.Items {seenItemIds := make(map[string]struct{})s.scanItem(ctx, chunksChan, collection, metadata, item, "", seenItemIds)@@ -385,11 +420,11 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch}func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string, seenItemIds map[string]struct{}) {- ctx = context.WithValue(ctx, "item_uid", item.Uid)+ ctx = context.WithValues(ctx, "item_uid", item.Uid)- ctx.Logger().V(3).Info("Starting to scan item",+ ctx.Logger().V(3).Info("Scanning item","item_parent_item_id", parentItemId,- "item_descendent_item_uids", fp.Map(func(i Item) string { return i.Uid })(item.Items),+ "item_count", len(item.Items),"item_event_count", len(item.Events),"item_response_count", len(item.Response),"item_variable_count", len(item.Variable),@@ -415,8 +450,9 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c// recurse through the foldersfor _, subItem := range item.Items {if _, ok := seenItemIds[subItem.Uid]; ok {- ctx.Logger().Info("Skipping already-seen item",+ ctx.Logger().V(4).Info("Skipping item","seen_item_id", subItem.Uid,+ "reason", "already seen",)continue}@@ -461,7 +497,7 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, cs.scanAuth(ctx, chunksChan, metadata, item.Auth, item.Request.URL)metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN- ctx.Logger().V(3).Info("Finished scanning item", "item_uid", item.Uid)+ ctx.Logger().V(5).Info("Finished scanning item")}func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, event Event) {@@ -486,7 +522,7 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT}- ctx = context.WithValue(ctx, "event_listen", event.Listen)+ ctx = context.WithValues(ctx, "event_listen", event.Listen)s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, data, DefaultMaxRecursionDepth)), metadata)metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN}@@ -540,14 +576,15 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, mif u.Raw != "" {parsedURL, err := url.Parse(u.Raw)if err != nil {- ctx.Logger().V(2).Info("error parsing URL in basic auth check", "url", u.Raw)+ // the URL is not logged, it can hold credentials+ ctx.Logger().V(4).Info("Skipping basic auth", "reason", "URL could not be parsed")return}parsedURL.User = url.User(username + ":" + password)decodedURL, err := url.PathUnescape(parsedURL.String())if err != nil {- ctx.Logger().V(2).Info("error parsing URL in basic auth check", "url", u.Raw)+ ctx.Logger().V(4).Info("Skipping basic auth", "reason", "URL could not be decoded")return}authData += decodedURL@@ -590,9 +627,9 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m}func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, r Request) {- ctx.Logger().V(3).Info("scanning http request",+ ctx.Logger().V(5).Info("Scanning HTTP request","request_header_count", len(r.HeaderKeyValue),- "request_has_string_header", r.HeaderString == nil,+ "request_has_string_header", r.HeaderString != nil,"request_url_query_param_count", len(r.URL.Query),"request_url_path_param_count", len(r.URL.Path),)@@ -650,7 +687,8 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C}func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, b Body) {- ctx.Logger().V(3).Info("scanning request body",+ ctx.Logger().V(5).Info("Scanning request body",+ "mode", b.Mode,"request_body_form_data_count", len(b.FormData),"request_body_url_encoded_param_count", len(b.URLEncoded),)@@ -733,7 +771,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, variableData VariableData) {if len(variableData.KeyValues) == 0 {- ctx.Logger().V(2).Info("no variables to scan", "type", m.Type, "item_uuid", m.FullID)+ ctx.Logger().V(5).Info("No variables to scan", "type", m.Type, "item_uuid", m.FullID)return}@@ -769,14 +807,14 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) {if data == "" {- ctx.Logger().V(3).Info("Data string is empty", "workspace_id", metadata.WorkspaceUUID)+ ctx.Logger().V(5).Info("Data string is empty", "type", metadata.Type)return}if metadata.FieldType == "" {metadata.FieldType = metadata.Type}- ctx.Logger().V(3).Info("Generating chunk and passing it to the channel", "link", metadata.Link)+ ctx.Logger().V(5).Info("Sending chunk", "link", metadata.Link, "field_type", metadata.FieldType)chunksChan <- &sources.Chunk{SourceType: s.Type(),SourceName: s.name,diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.goindex dac2550ba..04a4560b2 100644--- a/pkg/sources/postman/postman_client.go+++ b/pkg/sources/postman/postman_client.go@@ -225,7 +225,7 @@ func NewClient(postmanToken string, metrics *metrics) *Client {// Requests for large objects (usually collections) take a long time. While we don't think that _every_// request will take this long, some might take 5 seconds or more. This seems reasonable, but we should// be very cautious about bumping it further- HTTPClient: common.RetryableHTTPClientTimeout(30),+ HTTPClient: common.RetryableHTTPClientTimeout(30, withAPILogHooks()),Headers: bh,WorkspaceAndCollectionRateLimiter: rate.NewLimiter(rate.Every(time.Second), 1),GeneralRateLimiter: rate.NewLimiter(rate.Every(time.Second/5), 1),@@ -243,12 +243,13 @@ func (c *Client) NewRequest(ctx trContext.Context, urlStr string, headers map[streturn nil, err}- var h = c.Headers- for k, v := range headers {- h[k] = v+ // Set the per-request headers on the request only. Writing them into c.Headers+ // would send them on every later request too, and the client is shared by+ // requests that can run at the same time.+ for k, v := range c.Headers {+ req.Header.Set(k, v)}-- for k, v := range h {+ for k, v := range headers {req.Header.Set(k, v)}@@ -299,7 +300,7 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st// Don't log the response headers, they can hold cookies or tokens.// handleRateLimits logs the rate limit we care about.- ctx.Logger().V(4).Info("postman api response received", "status_code", resp.StatusCode)+ ctx.Logger().V(5).Info("Postman API response", "status_code", resp.StatusCode)if err := checkResponseStatus(resp); err != nil {return nil, err@@ -316,7 +317,7 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) errateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month")}if rateLimitRemainingMonthValue == "" {- ctx.Logger().V(2).Info("RateLimit-Remaining-Month header not found in response")+ ctx.Logger().V(5).Info("RateLimit-Remaining-Month header not found in response")return nil}@@ -337,7 +338,7 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) errateLimitTotalMonthValue = resp.Header.Get("X-RateLimit-Limit-Month")}if rateLimitTotalMonthValue == "" {- ctx.Logger().V(2).Info("RateLimit-Limit-Month header not found in response")+ ctx.Logger().V(5).Info("RateLimit-Limit-Month header not found in response")return nil}rateLimitTotalMonth, err := strconv.Atoi(rateLimitTotalMonthValue)@@ -353,13 +354,14 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er)if rateLimitTotalMonth == 0 {- ctx.Logger().V(2).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage")+ ctx.Logger().V(4).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage")return nil}// failsafe to abandon scan if we are over the threshold of monthly API requests usedpercentageUsed := float64(rateLimitTotalMonth-rateLimitRemainingMonth) / float64(rateLimitTotalMonth)if percentageUsed > abortScanAPIReqLimitThreshold {+ ctx.Logger().V(1).Info("Monthly API limit threshold reached, aborting scan", "percent_used", percentageUsed*100)return errAbortScanDueToAPIRateLimit}@@ -369,7 +371,7 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er// EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal).// Consider adding additional flags to support filtering.func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error) {- ctx.Logger().V(2).Info("enumerating workspaces")+ ctx.Logger().V(1).Info("Enumerating workspaces")workspacesObj := struct {Workspaces []Workspace `json:"workspaces"`}{}@@ -388,14 +390,17 @@ func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error)for i, workspace := range workspacesObj.Workspaces {tempWorkspace, err := c.GetWorkspace(ctx, workspace.Id)if err != nil {+ // Stop at the monthly API limit, like every other loop. Moving on would+ // spend more of the limit on the workspaces left.+ if errors.Is(err, errAbortScanDueToAPIRateLimit) {+ return nil, err+ }// Log and move on, because sometimes the Postman API seems to give us workspace IDs// that we don't have access to, so we don't want to kill the scan because of it.- ctx.Logger().Error(err, "could not get workspace %q (%s) during enumeration", workspace.Name, workspace.Id)+ ctx.Logger().Error(err, "Could not get workspace during enumeration", "workspace_id", workspace.Id, "workspace_name", workspace.Name)continue}workspacesObj.Workspaces[i] = tempWorkspace-- ctx.Logger().V(3).Info("individual workspace getting added to the slice", "workspace", workspacesObj.Workspaces[i])}return workspacesObj.Workspaces, nil@@ -403,7 +408,7 @@ func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error)// GetWorkspace returns the workspace for a given workspacefunc (c *Client) GetWorkspace(ctx trContext.Context, workspaceUUID string) (Workspace, error) {- ctx.Logger().V(2).Info("getting workspace", "workspace", workspaceUUID)+ ctx.Logger().V(5).Info("Fetching workspace", "workspace_id", workspaceUUID)obj := struct {Workspace Workspace `json:"workspace"`}{}@@ -425,7 +430,7 @@ func (c *Client) GetWorkspace(ctx trContext.Context, workspaceUUID string) (Work// GetEnvironmentVariables returns the environment variables for a given environmentfunc (c *Client) GetEnvironmentVariables(ctx trContext.Context, environment_uuid string) (VariableData, error) {- ctx.Logger().V(3).Info("getting environment variables", "environment_uuid", environment_uuid)+ ctx.Logger().V(5).Info("Fetching environment", "environment_uuid", environment_uuid)obj := struct {VariableData VariableData `json:"environment"`}{}@@ -447,7 +452,7 @@ func (c *Client) GetEnvironmentVariables(ctx trContext.Context, environment_uuid// GetCollection returns the collection for a given collectionfunc (c *Client) GetCollection(ctx trContext.Context, collection_uuid string) (Collection, error) {- ctx.Logger().V(3).Info("getting collection", "collection_uuid", collection_uuid)+ ctx.Logger().V(5).Info("Fetching collection", "collection_uuid", collection_uuid)obj := struct {Collection Collection `json:"collection"`}{}diff --git a/pkg/sources/postman/postman_test.go b/pkg/sources/postman/postman_test.goindex fee173bb6..3cbb386df 100644--- a/pkg/sources/postman/postman_test.go+++ b/pkg/sources/postman/postman_test.go@@ -2,6 +2,7 @@ package postmanimport ("fmt"+ "net/http""reflect""sort""strings"@@ -9,9 +10,16 @@ import ("time""github.com/go-errors/errors"+ "github.com/go-logr/zapr"+ "github.com/hashicorp/go-retryablehttp""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "go.uber.org/zap"+ "go.uber.org/zap/zapcore"+ "go.uber.org/zap/zaptest/observer""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/log/logtest""gopkg.in/h2non/gock.v1""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"@@ -998,3 +1006,210 @@ func TestSource_AbortScanRateLimitBelowThreshold(t *testing.T) {t.Errorf("expected no error, got: %v", err)}}++func TestSource_DebugLogging(t *testing.T) {+ const (+ testToken = "postmandebuglogtesttoken"+ testSecret = "postmandebuglogtestsecret"+ )++ mockAPI := func() {+ // the first workspace request fails, so the retry lines show up+ gock.New("https://api.getpostman.com").+ Get("/workspaces/ws1").+ Reply(503)+ gock.New("https://api.getpostman.com").+ Get("/workspaces/ws1").+ Reply(200).+ JSON(map[string]any{"workspace": map[string]any{+ "id": "ws1",+ "name": "Debug Workspace",+ "collections": []map[string]any{+ {"id": "col1", "name": "Keep", "uid": "u-col1"},+ {"id": "col2", "name": "Skip", "uid": "u-col2"},+ },+ "environments": []map[string]any{+ {"id": "env1", "name": "Keep", "uid": "u-env1"},+ {"id": "env2", "name": "Skip", "uid": "u-env2"},+ },+ }})+ gock.New("https://api.getpostman.com").+ Get("/environments/u-env1").+ Reply(200).+ JSON(map[string]any{"environment": map[string]any{+ "id": "env1",+ "name": "Keep",+ "values": []map[string]any{{"key": "api_key", "value": testSecret, "enabled": true}},+ }})+ gock.New("https://api.getpostman.com").+ Get("/collections/u-col1").+ Reply(200).+ JSON(map[string]any{"collection": map[string]any{+ "info": map[string]any{"_postman_id": "col1", "name": "Keep", "uid": "u-col1"},+ "item": []map[string]any{{+ "id": "i1",+ "uid": "u-i1",+ "name": "folder",+ "item": []map[string]any{+ // a folder that points back at itself is skipped, not scanned again+ {"id": "i1", "uid": "u-i1", "name": "folder"},+ {+ "id": "i2",+ "uid": "u-i2",+ "name": "request",+ "request": map[string]any{+ "method": "GET",+ "url": map[string]any{+ "raw": "https://example.com/v1/users",+ "protocol": "https",+ "host": []string{"example", "com"},+ "path": []string{"v1", "users"},+ },+ },+ },+ },+ }},+ }})+ }++ runSource := func(t *testing.T, ctx context.Context) {+ t.Helper()+ defer gock.Off()+ mockAPI()++ s, conn := createTestSource(&sourcespb.Postman{+ Credential: &sourcespb.Postman_Token{Token: testToken},+ Workspaces: []string{"ws1"},+ ExcludeCollections: []string{"u-col2"},+ ExcludeEnvironments: []string{"u-env2"},+ })+ // data is only sent with a detector keyword next to it, the engine sets these+ s.DetectorKeywords = map[string]struct{}{"api_key": {}}+ require.NoError(t, s.Init(ctx, "test debug logging", 1, 1, false, conn, 1))++ // intercept the client inside the retryable client, so the retry and response+ // hooks still run+ inner := s.client.HTTPClient.Transport.(*retryablehttp.RoundTripper).Client.HTTPClient+ gock.InterceptClient(inner)+ defer gock.RestoreClient(inner)++ chunksCh := make(chan *sources.Chunk, 100)+ require.NoError(t, s.Chunks(ctx, chunksCh))+ close(chunksCh)++ var found bool+ for chunk := range chunksCh {+ if strings.Contains(string(chunk.Data), testSecret) {+ found = true+ }+ }+ assert.True(t, found, "secret was not scanned")+ assert.True(t, gock.IsDone(), "pending mocks: %v", gock.Pending())+ }++ t.Run("debug on", func(t *testing.T) {+ ctx, logs := logtest.DebugContext()+ ctx = context.WithValues(ctx, "source_name", "test debug logging", "job_id", 1)+ runSource(t, ctx)++ // lines from each level show up+ for _, msg := range []string{+ "Postman source initialized", // V(1)+ "Scanning workspace", // V(2)+ "Scanning collection", // V(2)+ "Scanning environment", // V(3)+ "Scanning item", // V(3)+ "Skipping collection", // V(4)+ "Skipping environment", // V(4)+ "Skipping item", // V(4)+ "Postman API request failed", // V(4)+ "Retrying Postman API request", // V(4)+ "Fetching workspace", // V(5)+ "Postman API response", // V(5)+ "Sending chunk", // V(5)+ } {+ assert.NotZero(t, logs.FilterMessage(msg).Len(), "missing log line %q", msg)+ }++ failed := logs.FilterMessage("Postman API request failed").All()+ require.Len(t, failed, 1)+ assert.Equal(t, "/workspaces/ws1", failed[0].ContextMap()["path"])+ assert.EqualValues(t, http.StatusServiceUnavailable, failed[0].ContextMap()["status"])++ for _, msg := range []string{"Skipping collection", "Skipping environment"} {+ skips := logs.FilterMessage(msg).All()+ require.Len(t, skips, 1, msg)+ assert.Equal(t, "matches filter", skips[0].ContextMap()["reason"])+ }+ skips := logs.FilterMessage("Skipping item").All()+ require.Len(t, skips, 1)+ assert.Equal(t, "already seen", skips[0].ContextMap()["reason"])++ // every line came from the context logger, so it carries the source fields,+ // and none of them contain the token or the secret+ for _, entry := range logs.All() {+ fields := entry.ContextMap()+ assert.Contains(t, fields, "source_name", "line %q", entry.Message)+ assert.Contains(t, fields, "job_id", "line %q", entry.Message)+ for _, value := range []string{testToken, testSecret} {+ assert.NotContains(t, entry.Message, value)+ assert.NotContains(t, fmt.Sprint(fields), value)+ }+ }+ })++ t.Run("debug off", func(t *testing.T) {+ core, logs := observer.New(zapcore.InfoLevel)+ ctx := context.WithLogger(context.Background(), zapr.NewLogger(zap.New(core)))+ runSource(t, ctx)++ assert.Zero(t, logs.Len(), "unexpected lines: %v", logs.All())+ })+}++func TestClient_NewRequestKeepsSharedHeaders(t *testing.T) {+ c := NewClient("token", newMetrics("test"))++ req, err := c.NewRequest(context.Background(), "https://api.getpostman.com/workspaces", map[string]string{"X-Extra": "one"})+ require.NoError(t, err)+ assert.Equal(t, "one", req.Header.Get("X-Extra"))+ assert.Equal(t, "token", req.Header.Get("X-API-Key"))++ // the extra header must not leak into the next request+ assert.NotContains(t, c.Headers, "X-Extra")+ req, err = c.NewRequest(context.Background(), "https://api.getpostman.com/workspaces", nil)+ require.NoError(t, err)+ assert.Empty(t, req.Header.Get("X-Extra"))+}++func TestSource_EnumerateWorkspacesStopsAtMonthlyLimit(t *testing.T) {+ defer gock.Off()+ gock.New("https://api.getpostman.com").+ Get("/workspaces").+ Reply(200).+ BodyString(`{"workspaces":[{"id":"0","name":"workspace-0"},{"id":"1","name":"workspace-1"}]}`)+ // 90% of the monthly limit is used, which is over the abort threshold+ gock.New("https://api.getpostman.com").+ Get("/workspaces/0").+ Reply(200).+ SetHeader("RateLimit-Remaining-Month", "10").+ SetHeader("RateLimit-Limit-Month", "100").+ BodyString(`{"workspace":{"id":"0","name":"workspace-0"}}`)+ gock.New("https://api.getpostman.com").+ Get("/workspaces/1").+ Reply(200).+ BodyString(`{"workspace":{"id":"1","name":"workspace-1"}}`)++ ctx := context.Background()+ s, conn := createTestSource(&sourcespb.Postman{+ Credential: &sourcespb.Postman_Token{Token: "super-secret-token"},+ })+ require.NoError(t, s.Init(ctx, "test - postman", 0, 1, false, conn, 1))+ gock.InterceptClient(s.client.HTTPClient)+ defer gock.RestoreClient(s.client.HTTPClient)++ _, err := s.client.EnumerateWorkspaces(ctx)+ assert.ErrorIs(t, err, errAbortScanDueToAPIRateLimit)+ // the second workspace was never requested+ assert.Len(t, gock.Pending(), 1)+}diff --git a/pkg/sources/postman/substitution.go b/pkg/sources/postman/substitution.goindex 491ac119c..be63f9be1 100644--- a/pkg/sources/postman/substitution.go+++ b/pkg/sources/postman/substitution.go@@ -89,7 +89,7 @@ func (s *Source) buildSubstitution() {// Limit recursion depth to prevent stack overflowif depth > maxRecursionDepth {- ctx.Logger().V(2).Info("Limited recursion depth",+ ctx.Logger().V(4).Info("Limited recursion depth","depth", depth,)combos[data] = struct{}{}