Skip to content

Find, verify, and analyze leaked credentials

Clean up Postman source logging for per-source debug (#5510)

Kashif Khan98e399ef21e8
diff --git a/pkg/sources/postman/README.md b/pkg/sources/postman/README.md
index d8df89122..bd1db403e 100644
--- a/pkg/sources/postman/README.md
+++ b/pkg/sources/postman/README.md
@@ -222,6 +222,18 @@ trufflehog postman --token PMAK-xxxxxxxxxxxxxxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxx
trufflehog postman --collection-paths /path/to/collection.json --environment-paths /path/to/environment.json
```
+## Debug Logs
+
+With debug on for this source (`debug: true` in the config), the scan logs:
+
+- `V(1)`: `Postman source initialized` with the config choices (the credential type only, never the token), the start and end of workspace enumeration, and `Monthly API limit threshold reached, aborting scan` when the monthly API limit stops a scan.
+- `V(2)`: `Scanning workspace`, `Scanning collection` and `Scanning local file`, with their counts.
+- `V(3)`: `Scanning environment` and `Scanning item` for each environment and each folder or request.
+- `V(4)`: `Skipping collection`, `Skipping environment`, `Skipping item` and `Skipping basic auth` with a `reason`, `Postman API request failed` for any API response of 400 or above (with the status, path and `Retry-After` header), `Retrying Postman API request` for each retry, and `Limited recursion depth` when variable substitution stops early.
+- `V(5)`: each API request and response status, the IDs inside a workspace or collection, request details, and `Sending chunk` for each chunk sent to the detectors.
+
+Lines carry `workspace_id`, `collection_uuid`, `item_uid` and `event_listen` where they apply. Data is only sent to the detectors when a detector keyword is found nearby, so a field with no keyword logs `Data string is empty` at `V(5)`.
+
## Troubleshooting
### Common Issues
diff --git a/pkg/sources/postman/apilog.go b/pkg/sources/postman/apilog.go
new file mode 100644
index 000000000..06eab9fdb
--- /dev/null
+++ b/pkg/sources/postman/apilog.go
@@ -0,0 +1,46 @@
+package postman
+
+import (
+ "net/http"
+
+ "github.com/hashicorp/go-retryablehttp"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+// withAPILogHooks logs retries and failed responses from the Postman API. The client
+// retries 429 and 5xx on its own, so without these hooks support can't see them. The
+// hooks log through the request's context, so every request must carry the scan ctx.
+func withAPILogHooks() common.ClientOption {
+ return func(c *retryablehttp.Client) {
+ c.RequestLogHook = logAPIRetry
+ c.ResponseLogHook = logAPIResponse
+ }
+}
+
+// logAPIRetry is a retryablehttp.RequestLogHook. It runs before every attempt and
+// logs the ones that are retries.
+func logAPIRetry(_ retryablehttp.Logger, req *http.Request, attempt int) {
+ if attempt == 0 {
+ return
+ }
+ trContext.AddLogger(req.Context()).Logger().V(4).Info("Retrying Postman API request",
+ "path", req.URL.Path,
+ "attempt", attempt,
+ )
+}
+
+// logAPIResponse is a retryablehttp.ResponseLogHook. It logs failed responses,
+// including the ones that will be retried. Headers are not logged, they can hold
+// cookies or tokens.
+func logAPIResponse(_ retryablehttp.Logger, resp *http.Response) {
+ if resp.StatusCode < 400 {
+ return
+ }
+ trContext.AddLogger(resp.Request.Context()).Logger().V(4).Info("Postman API request failed",
+ "path", resp.Request.URL.Path,
+ "status", resp.StatusCode,
+ "retry_after", resp.Header.Get("Retry-After"),
+ )
+}
diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.go
index c0a7fccc1..95908e5e0 100644
--- a/pkg/sources/postman/postman.go
+++ b/pkg/sources/postman/postman.go
@@ -128,9 +128,34 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
return errors.New("credential type not implemented for Postman")
}
+ ctx.Logger().V(1).Info("Postman source initialized",
+ "auth_method", authMethod(&conn),
+ "workspaces", len(conn.Workspaces),
+ "collections", len(conn.Collections),
+ "environments", len(conn.Environments),
+ "include_collections", len(conn.IncludeCollections),
+ "exclude_collections", len(conn.ExcludeCollections),
+ "include_environments", len(conn.IncludeEnvironments),
+ "exclude_environments", len(conn.ExcludeEnvironments),
+ "local_workspaces", len(conn.WorkspacePaths),
+ "local_collections", len(conn.CollectionPaths),
+ "local_environments", len(conn.EnvironmentPaths),
+ )
return nil
}
+// authMethod names the credential type for logs. It never returns the credential.
+func authMethod(conn *sourcespb.Postman) string {
+ switch conn.Credential.(type) {
+ case *sourcespb.Postman_Token:
+ return "token"
+ case *sourcespb.Postman_Unauthenticated:
+ return "unauthenticated"
+ default:
+ return "unknown"
+ }
+}
+
// Chunks scans the Postman source and sends the data to the chunks chan.
// It scans the local environment, collection, and workspace files, and then scans the Postman API if a token is provided.
// The Postman source is different to our other sources in that we are not relying on the data we read from the source to contain
@@ -142,6 +167,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
// Scan local environments
for _, envPath := range s.conn.EnvironmentPaths {
+ ctx.Logger().V(2).Info("Scanning local file", "kind", "environment", "path", envPath)
env := VariableData{}
contents, err := os.ReadFile(envPath)
if err != nil {
@@ -155,6 +181,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
// Scan local collections
for _, collectionPath := range s.conn.CollectionPaths {
+ ctx.Logger().V(2).Info("Scanning local file", "kind", "collection", "path", collectionPath)
collection := Collection{}
contents, err := os.ReadFile(collectionPath)
if err != nil {
@@ -168,6 +195,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
// Scan local workspaces
for _, workspacePath := range s.conn.WorkspacePaths {
+ ctx.Logger().V(2).Info("Scanning local file", "kind", "workspace", "path", workspacePath)
// check if zip file
workspace := Workspace{}
if strings.HasSuffix(workspacePath, ".zip") {
@@ -191,11 +219,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
}
// Log and move on, because sometimes the Postman API seems to give us workspace IDs
// that we don't have access to, so we don't want to kill the scan because of it.
- ctx.Logger().Error(err, "error getting workspace %s", workspaceID)
+ ctx.Logger().Error(err, "Error getting workspace", "workspace_id", workspaceID)
continue
}
s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspaceID), "")
- ctx.Logger().V(2).Info("scanning workspace from workspaces given", "workspace", workspaceID)
if err = s.scanWorkspace(ctx, chunksChan, w); err != nil {
return fmt.Errorf("error scanning workspace %s: %w", workspaceID, err)
}
@@ -204,6 +231,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
// Scan collections
for _, collectionID := range s.conn.Collections {
if shouldSkip(collectionID, s.conn.IncludeCollections, s.conn.ExcludeCollections) {
+ ctx.Logger().V(4).Info("Skipping collection", "collection_uuid", collectionID, "reason", "matches filter")
continue
}
@@ -214,7 +242,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
}
// Log and move on, because sometimes the Postman API seems to give us collection IDs
// that we don't have access to, so we don't want to kill the scan because of it.
- ctx.Logger().Error(err, "error getting collection %s", collectionID)
+ ctx.Logger().Error(err, "Error getting collection", "collection_uuid", collectionID)
continue
}
s.SetProgressOngoing(fmt.Sprintf("Scanning collection %s", collectionID), "")
@@ -227,7 +255,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
if err != nil {
return fmt.Errorf("error enumerating postman workspaces: %w", err)
}
- ctx.Logger().V(2).Info("enumerated workspaces", "workspaces", workspaces)
+ ctx.Logger().V(1).Info("Enumerated workspaces", "count", len(workspaces))
for _, workspace := range workspaces {
s.SetProgressOngoing(fmt.Sprintf("Scanning workspace %s", workspace.Id), "")
if err = s.scanWorkspace(ctx, chunksChan, workspace); err != nil {
@@ -262,13 +290,19 @@ func (s *Source) scanLocalWorkspace(ctx context.Context, chunksChan chan *source
}
func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chunk, workspace Workspace) error {
- ctx.Logger().V(4).Info("scanning workspace",
- "workspace_id", workspace.Id,
- "collection_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Collections),
- "environment_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Environments),
- "collection_raw_uids", fp.Map(func(c Collection) string { return c.Info.Uid })(workspace.CollectionsRaw),
- "environment_raw_ids", fp.Map(func(v VariableData) string { return v.Id })(workspace.EnvironmentsRaw),
+ ctx = context.WithValues(ctx, "workspace_id", workspace.Id)
+ ctx.Logger().V(2).Info("Scanning workspace",
+ "collection_count", len(workspace.Collections),
+ "environment_count", len(workspace.Environments),
)
+ if ctx.Logger().V(5).Enabled() {
+ ctx.Logger().V(5).Info("Workspace contents",
+ "collection_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Collections),
+ "environment_uids", fp.Map(func(i IdNameUid) string { return i.Uid })(workspace.Environments),
+ "collection_raw_uids", fp.Map(func(c Collection) string { return c.Info.Uid })(workspace.CollectionsRaw),
+ "environment_raw_ids", fp.Map(func(v VariableData) string { return v.Id })(workspace.EnvironmentsRaw),
+ )
+ }
// reset keywords for each workspace
s.resetKeywords()
s.attemptToAddKeyword(workspace.Name)
@@ -283,6 +317,10 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
// gather and scan environment variables
for _, envID := range workspace.Environments {
+ if shouldSkip(envID.Uid, s.conn.IncludeEnvironments, s.conn.ExcludeEnvironments) {
+ ctx.Logger().V(4).Info("Skipping environment", "environment_uuid", envID.Uid, "reason", "matches filter")
+ continue
+ }
envVars, err := s.client.GetEnvironmentVariables(ctx, envID.Uid)
if err != nil {
if errors.Is(err, errAbortScanDueToAPIRateLimit) {
@@ -291,16 +329,13 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
ctx.Logger().Error(err, "could not get env variables", "environment_uuid", envID.Uid)
continue
}
- if shouldSkip(envID.Uid, s.conn.IncludeEnvironments, s.conn.ExcludeEnvironments) {
- continue
- }
metadata.Type = ENVIRONMENT_TYPE
metadata.Link = LINK_BASE_URL + "environments/" + envID.Uid
metadata.FullID = envVars.Id
metadata.EnvironmentID = envID.Uid
metadata.EnvironmentName = envVars.Name
- ctx.Logger().V(2).Info("scanning environment vars", "environment_uuid", metadata.FullID)
+ ctx.Logger().V(3).Info("Scanning environment", "environment_uuid", envID.Uid, "variable_count", len(envVars.KeyValues))
for _, word := range strings.Split(envVars.Name, " ") {
s.attemptToAddKeyword(word)
}
@@ -312,15 +347,14 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
metadata.FullID = ""
metadata.EnvironmentID = ""
metadata.EnvironmentName = ""
- ctx.Logger().V(2).Info("finished scanning environment vars", "environment_uuid", metadata.FullID)
}
- ctx.Logger().V(2).Info("finished scanning environments")
// scan all the collections in the workspace.
// at this point we have all the possible
// substitutions from Environment variables
for _, collectionID := range workspace.Collections {
if shouldSkip(collectionID.Uid, s.conn.IncludeCollections, s.conn.ExcludeCollections) {
+ ctx.Logger().V(4).Info("Skipping collection", "collection_uuid", collectionID.Uid, "reason", "matches filter")
continue
}
collection, err := s.client.GetCollection(ctx, collectionID.Uid)
@@ -330,7 +364,7 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
}
// Log and move on, because sometimes the Postman API seems to give us collection IDs
// that we don't have access to, so we don't want to kill the scan because of it.
- ctx.Logger().Error(err, "error getting collection %s", collectionID)
+ ctx.Logger().Error(err, "Error getting collection", "collection_uuid", collectionID.Uid)
continue
}
s.scanCollection(ctx, chunksChan, metadata, collection)
@@ -345,9 +379,16 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
"collection_name", collection.Info.Name,
"collection_uuid", collection.Info.Uid,
)
- ctx.Logger().V(2).Info("starting to scan collection",
+ ctx.Logger().V(2).Info("Scanning collection",
"variable_count", len(collection.Variables),
+ "event_count", len(collection.Events),
+ "item_count", len(collection.Items),
)
+ if ctx.Logger().V(5).Enabled() {
+ ctx.Logger().V(5).Info("Collection items",
+ "item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items),
+ )
+ }
metadata.CollectionInfo = collection.Info
metadata.Type = COLLECTION_TYPE
s.attemptToAddKeyword(collection.Info.Name)
@@ -368,16 +409,10 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
// collections don't have URLs in the Postman API, but we can scan the Authorization section without it.
s.scanAuth(ctx, chunksChan, metadata, collection.Auth, URL{})
- ctx.Logger().V(3).Info("Scanning events in collection",
- "event_count", len(collection.Events),
- )
for _, event := range collection.Events {
s.scanEvent(ctx, chunksChan, metadata, event)
}
- ctx.Logger().V(3).Info("Scanning items in collection",
- "item_ids", fp.Map(func(i Item) string { return i.Id })(collection.Items),
- )
for _, item := range collection.Items {
seenItemIds := make(map[string]struct{})
s.scanItem(ctx, chunksChan, collection, metadata, item, "", seenItemIds)
@@ -385,11 +420,11 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
}
func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, collection Collection, metadata Metadata, item Item, parentItemId string, seenItemIds map[string]struct{}) {
- ctx = context.WithValue(ctx, "item_uid", item.Uid)
+ ctx = context.WithValues(ctx, "item_uid", item.Uid)
- ctx.Logger().V(3).Info("Starting to scan item",
+ ctx.Logger().V(3).Info("Scanning item",
"item_parent_item_id", parentItemId,
- "item_descendent_item_uids", fp.Map(func(i Item) string { return i.Uid })(item.Items),
+ "item_count", len(item.Items),
"item_event_count", len(item.Events),
"item_response_count", len(item.Response),
"item_variable_count", len(item.Variable),
@@ -415,8 +450,9 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
// recurse through the folders
for _, subItem := range item.Items {
if _, ok := seenItemIds[subItem.Uid]; ok {
- ctx.Logger().Info("Skipping already-seen item",
+ ctx.Logger().V(4).Info("Skipping item",
"seen_item_id", subItem.Uid,
+ "reason", "already seen",
)
continue
}
@@ -461,7 +497,7 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
s.scanAuth(ctx, chunksChan, metadata, item.Auth, item.Request.URL)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
- ctx.Logger().V(3).Info("Finished scanning item", "item_uid", item.Uid)
+ ctx.Logger().V(5).Info("Finished scanning item")
}
func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, event Event) {
@@ -486,7 +522,7 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,
metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
}
- ctx = context.WithValue(ctx, "event_listen", event.Listen)
+ ctx = context.WithValues(ctx, "event_listen", event.Listen)
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstituteSet(ctx, metadata, data, DefaultMaxRecursionDepth)), metadata)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}
@@ -540,14 +576,15 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
if u.Raw != "" {
parsedURL, err := url.Parse(u.Raw)
if err != nil {
- ctx.Logger().V(2).Info("error parsing URL in basic auth check", "url", u.Raw)
+ // the URL is not logged, it can hold credentials
+ ctx.Logger().V(4).Info("Skipping basic auth", "reason", "URL could not be parsed")
return
}
parsedURL.User = url.User(username + ":" + password)
decodedURL, err := url.PathUnescape(parsedURL.String())
if err != nil {
- ctx.Logger().V(2).Info("error parsing URL in basic auth check", "url", u.Raw)
+ ctx.Logger().V(4).Info("Skipping basic auth", "reason", "URL could not be decoded")
return
}
authData += decodedURL
@@ -590,9 +627,9 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
}
func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, r Request) {
- ctx.Logger().V(3).Info("scanning http request",
+ ctx.Logger().V(5).Info("Scanning HTTP request",
"request_header_count", len(r.HeaderKeyValue),
- "request_has_string_header", r.HeaderString == nil,
+ "request_has_string_header", r.HeaderString != nil,
"request_url_query_param_count", len(r.URL.Query),
"request_url_path_param_count", len(r.URL.Path),
)
@@ -650,7 +687,8 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
}
func (s *Source) scanRequestBody(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, b Body) {
- ctx.Logger().V(3).Info("scanning request body",
+ ctx.Logger().V(5).Info("Scanning request body",
+ "mode", b.Mode,
"request_body_form_data_count", len(b.FormData),
"request_body_url_encoded_param_count", len(b.URLEncoded),
)
@@ -733,7 +771,7 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, variableData VariableData) {
if len(variableData.KeyValues) == 0 {
- ctx.Logger().V(2).Info("no variables to scan", "type", m.Type, "item_uuid", m.FullID)
+ ctx.Logger().V(5).Info("No variables to scan", "type", m.Type, "item_uuid", m.FullID)
return
}
@@ -769,14 +807,14 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) {
if data == "" {
- ctx.Logger().V(3).Info("Data string is empty", "workspace_id", metadata.WorkspaceUUID)
+ ctx.Logger().V(5).Info("Data string is empty", "type", metadata.Type)
return
}
if metadata.FieldType == "" {
metadata.FieldType = metadata.Type
}
- ctx.Logger().V(3).Info("Generating chunk and passing it to the channel", "link", metadata.Link)
+ ctx.Logger().V(5).Info("Sending chunk", "link", metadata.Link, "field_type", metadata.FieldType)
chunksChan <- &sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.go
index dac2550ba..04a4560b2 100644
--- a/pkg/sources/postman/postman_client.go
+++ b/pkg/sources/postman/postman_client.go
@@ -225,7 +225,7 @@ func NewClient(postmanToken string, metrics *metrics) *Client {
// Requests for large objects (usually collections) take a long time. While we don't think that _every_
// request will take this long, some might take 5 seconds or more. This seems reasonable, but we should
// be very cautious about bumping it further
- HTTPClient: common.RetryableHTTPClientTimeout(30),
+ HTTPClient: common.RetryableHTTPClientTimeout(30, withAPILogHooks()),
Headers: bh,
WorkspaceAndCollectionRateLimiter: rate.NewLimiter(rate.Every(time.Second), 1),
GeneralRateLimiter: rate.NewLimiter(rate.Every(time.Second/5), 1),
@@ -243,12 +243,13 @@ func (c *Client) NewRequest(ctx trContext.Context, urlStr string, headers map[st
return nil, err
}
- var h = c.Headers
- for k, v := range headers {
- h[k] = v
+ // Set the per-request headers on the request only. Writing them into c.Headers
+ // would send them on every later request too, and the client is shared by
+ // requests that can run at the same time.
+ for k, v := range c.Headers {
+ req.Header.Set(k, v)
}
-
- for k, v := range h {
+ for k, v := range headers {
req.Header.Set(k, v)
}
@@ -299,7 +300,7 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString st
// Don't log the response headers, they can hold cookies or tokens.
// handleRateLimits logs the rate limit we care about.
- ctx.Logger().V(4).Info("postman api response received", "status_code", resp.StatusCode)
+ ctx.Logger().V(5).Info("Postman API response", "status_code", resp.StatusCode)
if err := checkResponseStatus(resp); err != nil {
return nil, err
@@ -316,7 +317,7 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er
rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month")
}
if rateLimitRemainingMonthValue == "" {
- ctx.Logger().V(2).Info("RateLimit-Remaining-Month header not found in response")
+ ctx.Logger().V(5).Info("RateLimit-Remaining-Month header not found in response")
return nil
}
@@ -337,7 +338,7 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er
rateLimitTotalMonthValue = resp.Header.Get("X-RateLimit-Limit-Month")
}
if rateLimitTotalMonthValue == "" {
- ctx.Logger().V(2).Info("RateLimit-Limit-Month header not found in response")
+ ctx.Logger().V(5).Info("RateLimit-Limit-Month header not found in response")
return nil
}
rateLimitTotalMonth, err := strconv.Atoi(rateLimitTotalMonthValue)
@@ -353,13 +354,14 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er
)
if rateLimitTotalMonth == 0 {
- ctx.Logger().V(2).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage")
+ ctx.Logger().V(4).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage")
return nil
}
// failsafe to abandon scan if we are over the threshold of monthly API requests used
percentageUsed := float64(rateLimitTotalMonth-rateLimitRemainingMonth) / float64(rateLimitTotalMonth)
if percentageUsed > abortScanAPIReqLimitThreshold {
+ ctx.Logger().V(1).Info("Monthly API limit threshold reached, aborting scan", "percent_used", percentageUsed*100)
return errAbortScanDueToAPIRateLimit
}
@@ -369,7 +371,7 @@ func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) er
// EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal).
// Consider adding additional flags to support filtering.
func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error) {
- ctx.Logger().V(2).Info("enumerating workspaces")
+ ctx.Logger().V(1).Info("Enumerating workspaces")
workspacesObj := struct {
Workspaces []Workspace `json:"workspaces"`
}{}
@@ -388,14 +390,17 @@ func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error)
for i, workspace := range workspacesObj.Workspaces {
tempWorkspace, err := c.GetWorkspace(ctx, workspace.Id)
if err != nil {
+ // Stop at the monthly API limit, like every other loop. Moving on would
+ // spend more of the limit on the workspaces left.
+ if errors.Is(err, errAbortScanDueToAPIRateLimit) {
+ return nil, err
+ }
// Log and move on, because sometimes the Postman API seems to give us workspace IDs
// that we don't have access to, so we don't want to kill the scan because of it.
- ctx.Logger().Error(err, "could not get workspace %q (%s) during enumeration", workspace.Name, workspace.Id)
+ ctx.Logger().Error(err, "Could not get workspace during enumeration", "workspace_id", workspace.Id, "workspace_name", workspace.Name)
continue
}
workspacesObj.Workspaces[i] = tempWorkspace
-
- ctx.Logger().V(3).Info("individual workspace getting added to the slice", "workspace", workspacesObj.Workspaces[i])
}
return workspacesObj.Workspaces, nil
@@ -403,7 +408,7 @@ func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error)
// GetWorkspace returns the workspace for a given workspace
func (c *Client) GetWorkspace(ctx trContext.Context, workspaceUUID string) (Workspace, error) {
- ctx.Logger().V(2).Info("getting workspace", "workspace", workspaceUUID)
+ ctx.Logger().V(5).Info("Fetching workspace", "workspace_id", workspaceUUID)
obj := struct {
Workspace Workspace `json:"workspace"`
}{}
@@ -425,7 +430,7 @@ func (c *Client) GetWorkspace(ctx trContext.Context, workspaceUUID string) (Work
// GetEnvironmentVariables returns the environment variables for a given environment
func (c *Client) GetEnvironmentVariables(ctx trContext.Context, environment_uuid string) (VariableData, error) {
- ctx.Logger().V(3).Info("getting environment variables", "environment_uuid", environment_uuid)
+ ctx.Logger().V(5).Info("Fetching environment", "environment_uuid", environment_uuid)
obj := struct {
VariableData VariableData `json:"environment"`
}{}
@@ -447,7 +452,7 @@ func (c *Client) GetEnvironmentVariables(ctx trContext.Context, environment_uuid
// GetCollection returns the collection for a given collection
func (c *Client) GetCollection(ctx trContext.Context, collection_uuid string) (Collection, error) {
- ctx.Logger().V(3).Info("getting collection", "collection_uuid", collection_uuid)
+ ctx.Logger().V(5).Info("Fetching collection", "collection_uuid", collection_uuid)
obj := struct {
Collection Collection `json:"collection"`
}{}
diff --git a/pkg/sources/postman/postman_test.go b/pkg/sources/postman/postman_test.go
index fee173bb6..3cbb386df 100644
--- a/pkg/sources/postman/postman_test.go
+++ b/pkg/sources/postman/postman_test.go
@@ -2,6 +2,7 @@ package postman
import (
"fmt"
+ "net/http"
"reflect"
"sort"
"strings"
@@ -9,9 +10,16 @@ import (
"time"
"github.com/go-errors/errors"
+ "github.com/go-logr/zapr"
+ "github.com/hashicorp/go-retryablehttp"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log/logtest"
"gopkg.in/h2non/gock.v1"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -998,3 +1006,210 @@ func TestSource_AbortScanRateLimitBelowThreshold(t *testing.T) {
t.Errorf("expected no error, got: %v", err)
}
}
+
+func TestSource_DebugLogging(t *testing.T) {
+ const (
+ testToken = "postmandebuglogtesttoken"
+ testSecret = "postmandebuglogtestsecret"
+ )
+
+ mockAPI := func() {
+ // the first workspace request fails, so the retry lines show up
+ gock.New("https://api.getpostman.com").
+ Get("/workspaces/ws1").
+ Reply(503)
+ gock.New("https://api.getpostman.com").
+ Get("/workspaces/ws1").
+ Reply(200).
+ JSON(map[string]any{"workspace": map[string]any{
+ "id": "ws1",
+ "name": "Debug Workspace",
+ "collections": []map[string]any{
+ {"id": "col1", "name": "Keep", "uid": "u-col1"},
+ {"id": "col2", "name": "Skip", "uid": "u-col2"},
+ },
+ "environments": []map[string]any{
+ {"id": "env1", "name": "Keep", "uid": "u-env1"},
+ {"id": "env2", "name": "Skip", "uid": "u-env2"},
+ },
+ }})
+ gock.New("https://api.getpostman.com").
+ Get("/environments/u-env1").
+ Reply(200).
+ JSON(map[string]any{"environment": map[string]any{
+ "id": "env1",
+ "name": "Keep",
+ "values": []map[string]any{{"key": "api_key", "value": testSecret, "enabled": true}},
+ }})
+ gock.New("https://api.getpostman.com").
+ Get("/collections/u-col1").
+ Reply(200).
+ JSON(map[string]any{"collection": map[string]any{
+ "info": map[string]any{"_postman_id": "col1", "name": "Keep", "uid": "u-col1"},
+ "item": []map[string]any{{
+ "id": "i1",
+ "uid": "u-i1",
+ "name": "folder",
+ "item": []map[string]any{
+ // a folder that points back at itself is skipped, not scanned again
+ {"id": "i1", "uid": "u-i1", "name": "folder"},
+ {
+ "id": "i2",
+ "uid": "u-i2",
+ "name": "request",
+ "request": map[string]any{
+ "method": "GET",
+ "url": map[string]any{
+ "raw": "https://example.com/v1/users",
+ "protocol": "https",
+ "host": []string{"example", "com"},
+ "path": []string{"v1", "users"},
+ },
+ },
+ },
+ },
+ }},
+ }})
+ }
+
+ runSource := func(t *testing.T, ctx context.Context) {
+ t.Helper()
+ defer gock.Off()
+ mockAPI()
+
+ s, conn := createTestSource(&sourcespb.Postman{
+ Credential: &sourcespb.Postman_Token{Token: testToken},
+ Workspaces: []string{"ws1"},
+ ExcludeCollections: []string{"u-col2"},
+ ExcludeEnvironments: []string{"u-env2"},
+ })
+ // data is only sent with a detector keyword next to it, the engine sets these
+ s.DetectorKeywords = map[string]struct{}{"api_key": {}}
+ require.NoError(t, s.Init(ctx, "test debug logging", 1, 1, false, conn, 1))
+
+ // intercept the client inside the retryable client, so the retry and response
+ // hooks still run
+ inner := s.client.HTTPClient.Transport.(*retryablehttp.RoundTripper).Client.HTTPClient
+ gock.InterceptClient(inner)
+ defer gock.RestoreClient(inner)
+
+ chunksCh := make(chan *sources.Chunk, 100)
+ require.NoError(t, s.Chunks(ctx, chunksCh))
+ close(chunksCh)
+
+ var found bool
+ for chunk := range chunksCh {
+ if strings.Contains(string(chunk.Data), testSecret) {
+ found = true
+ }
+ }
+ assert.True(t, found, "secret was not scanned")
+ assert.True(t, gock.IsDone(), "pending mocks: %v", gock.Pending())
+ }
+
+ t.Run("debug on", func(t *testing.T) {
+ ctx, logs := logtest.DebugContext()
+ ctx = context.WithValues(ctx, "source_name", "test debug logging", "job_id", 1)
+ runSource(t, ctx)
+
+ // lines from each level show up
+ for _, msg := range []string{
+ "Postman source initialized", // V(1)
+ "Scanning workspace", // V(2)
+ "Scanning collection", // V(2)
+ "Scanning environment", // V(3)
+ "Scanning item", // V(3)
+ "Skipping collection", // V(4)
+ "Skipping environment", // V(4)
+ "Skipping item", // V(4)
+ "Postman API request failed", // V(4)
+ "Retrying Postman API request", // V(4)
+ "Fetching workspace", // V(5)
+ "Postman API response", // V(5)
+ "Sending chunk", // V(5)
+ } {
+ assert.NotZero(t, logs.FilterMessage(msg).Len(), "missing log line %q", msg)
+ }
+
+ failed := logs.FilterMessage("Postman API request failed").All()
+ require.Len(t, failed, 1)
+ assert.Equal(t, "/workspaces/ws1", failed[0].ContextMap()["path"])
+ assert.EqualValues(t, http.StatusServiceUnavailable, failed[0].ContextMap()["status"])
+
+ for _, msg := range []string{"Skipping collection", "Skipping environment"} {
+ skips := logs.FilterMessage(msg).All()
+ require.Len(t, skips, 1, msg)
+ assert.Equal(t, "matches filter", skips[0].ContextMap()["reason"])
+ }
+ skips := logs.FilterMessage("Skipping item").All()
+ require.Len(t, skips, 1)
+ assert.Equal(t, "already seen", skips[0].ContextMap()["reason"])
+
+ // every line came from the context logger, so it carries the source fields,
+ // and none of them contain the token or the secret
+ for _, entry := range logs.All() {
+ fields := entry.ContextMap()
+ assert.Contains(t, fields, "source_name", "line %q", entry.Message)
+ assert.Contains(t, fields, "job_id", "line %q", entry.Message)
+ for _, value := range []string{testToken, testSecret} {
+ assert.NotContains(t, entry.Message, value)
+ assert.NotContains(t, fmt.Sprint(fields), value)
+ }
+ }
+ })
+
+ t.Run("debug off", func(t *testing.T) {
+ core, logs := observer.New(zapcore.InfoLevel)
+ ctx := context.WithLogger(context.Background(), zapr.NewLogger(zap.New(core)))
+ runSource(t, ctx)
+
+ assert.Zero(t, logs.Len(), "unexpected lines: %v", logs.All())
+ })
+}
+
+func TestClient_NewRequestKeepsSharedHeaders(t *testing.T) {
+ c := NewClient("token", newMetrics("test"))
+
+ req, err := c.NewRequest(context.Background(), "https://api.getpostman.com/workspaces", map[string]string{"X-Extra": "one"})
+ require.NoError(t, err)
+ assert.Equal(t, "one", req.Header.Get("X-Extra"))
+ assert.Equal(t, "token", req.Header.Get("X-API-Key"))
+
+ // the extra header must not leak into the next request
+ assert.NotContains(t, c.Headers, "X-Extra")
+ req, err = c.NewRequest(context.Background(), "https://api.getpostman.com/workspaces", nil)
+ require.NoError(t, err)
+ assert.Empty(t, req.Header.Get("X-Extra"))
+}
+
+func TestSource_EnumerateWorkspacesStopsAtMonthlyLimit(t *testing.T) {
+ defer gock.Off()
+ gock.New("https://api.getpostman.com").
+ Get("/workspaces").
+ Reply(200).
+ BodyString(`{"workspaces":[{"id":"0","name":"workspace-0"},{"id":"1","name":"workspace-1"}]}`)
+ // 90% of the monthly limit is used, which is over the abort threshold
+ gock.New("https://api.getpostman.com").
+ Get("/workspaces/0").
+ Reply(200).
+ SetHeader("RateLimit-Remaining-Month", "10").
+ SetHeader("RateLimit-Limit-Month", "100").
+ BodyString(`{"workspace":{"id":"0","name":"workspace-0"}}`)
+ gock.New("https://api.getpostman.com").
+ Get("/workspaces/1").
+ Reply(200).
+ BodyString(`{"workspace":{"id":"1","name":"workspace-1"}}`)
+
+ ctx := context.Background()
+ s, conn := createTestSource(&sourcespb.Postman{
+ Credential: &sourcespb.Postman_Token{Token: "super-secret-token"},
+ })
+ require.NoError(t, s.Init(ctx, "test - postman", 0, 1, false, conn, 1))
+ gock.InterceptClient(s.client.HTTPClient)
+ defer gock.RestoreClient(s.client.HTTPClient)
+
+ _, err := s.client.EnumerateWorkspaces(ctx)
+ assert.ErrorIs(t, err, errAbortScanDueToAPIRateLimit)
+ // the second workspace was never requested
+ assert.Len(t, gock.Pending(), 1)
+}
diff --git a/pkg/sources/postman/substitution.go b/pkg/sources/postman/substitution.go
index 491ac119c..be63f9be1 100644
--- a/pkg/sources/postman/substitution.go
+++ b/pkg/sources/postman/substitution.go
@@ -89,7 +89,7 @@ func (s *Source) buildSubstitution(
) {
// Limit recursion depth to prevent stack overflow
if depth > maxRecursionDepth {
- ctx.Logger().V(2).Info("Limited recursion depth",
+ ctx.Logger().V(4).Info("Limited recursion depth",
"depth", depth,
)
combos[data] = struct{}{}