Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[INS-204] Abort Postman scan if monthly API request limit crosses 80% (#4586)
Mustansir2d008a57d60e
diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.goindex 6831098dc..da904946a 100644--- a/pkg/sources/postman/postman.go+++ b/pkg/sources/postman/postman.go@@ -186,6 +186,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .for _, workspaceID := range s.conn.Workspaces {w, err := s.client.GetWorkspace(ctx, workspaceID)if err != nil {+ if errors.Is(err, errAbortScanDueToAPIRateLimit) {+ return err+ }// Log and move on, because sometimes the Postman API seems to give us workspace IDs// that we don't have access to, so we don't want to kill the scan because of it.ctx.Logger().Error(err, "error getting workspace %s", workspaceID)@@ -206,6 +209,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .collection, err := s.client.GetCollection(ctx, collectionID)if err != nil {+ if errors.Is(err, errAbortScanDueToAPIRateLimit) {+ return err+ }// Log and move on, because sometimes the Postman API seems to give us collection IDs// that we don't have access to, so we don't want to kill the scan because of it.ctx.Logger().Error(err, "error getting collection %s", collectionID)@@ -279,6 +285,9 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chufor _, envID := range workspace.Environments {envVars, err := s.client.GetEnvironmentVariables(ctx, envID.Uid)if err != nil {+ if errors.Is(err, errAbortScanDueToAPIRateLimit) {+ return err+ }ctx.Logger().Error(err, "could not get env variables", "environment_uuid", envID.Uid)continue}@@ -316,6 +325,9 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu}collection, err := s.client.GetCollection(ctx, collectionID.Uid)if err != nil {+ if errors.Is(err, errAbortScanDueToAPIRateLimit) {+ return err+ }// Log and move on, because sometimes the Postman API seems to give us collection IDs// that we don't have access to, so we don't want to kill the scan because of it.ctx.Logger().Error(err, "error getting collection %s", collectionID)diff --git a/pkg/sources/postman/postman_client.go b/pkg/sources/postman/postman_client.goindex 1562cc735..e41ee8440 100644--- a/pkg/sources/postman/postman_client.go+++ b/pkg/sources/postman/postman_client.go@@ -18,13 +18,16 @@ import ()const (- WORKSPACE_URL = "https://api.getpostman.com/workspaces/%s"- ENVIRONMENTS_URL = "https://api.getpostman.com/environments/%s"- COLLECTIONS_URL = "https://api.getpostman.com/collections/%s"- userAgent = "PostmanRuntime/7.26.8"- defaultContentType = "*"+ WORKSPACE_URL = "https://api.getpostman.com/workspaces/%s"+ ENVIRONMENTS_URL = "https://api.getpostman.com/environments/%s"+ COLLECTIONS_URL = "https://api.getpostman.com/collections/%s"+ userAgent = "PostmanRuntime/7.26.8"+ defaultContentType = "*"+ abortScanAPIReqLimitThreshold = 0.8 // Abort scan if 80% of monthly api request limit is reached)+var errAbortScanDueToAPIRateLimit = fmt.Errorf("aborting scan due to Postman API monthly requests limit being used over %f%%", abortScanAPIReqLimitThreshold*100)+type Workspace struct {Id string `json:"id"`Name string `json:"name"`@@ -284,22 +287,9 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString stc.Metrics.apiRequests.WithLabelValues(urlString).Inc()- rateLimitRemainingMonthValue := resp.Header.Get("RateLimit-Remaining-Month")- if rateLimitRemainingMonthValue == "" {- rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month")- }-- if rateLimitRemainingMonthValue != "" {- rateLimitRemainingMonth, err := strconv.Atoi(rateLimitRemainingMonthValue)- if err != nil {- ctx.Logger().Error(err, "Couldn't convert RateLimit-Remaining-Month to an int",- "header_value", rateLimitRemainingMonthValue,- )- } else {- c.Metrics.apiMonthlyRequestsRemaining.WithLabelValues().Set(- float64(rateLimitRemainingMonth),- )- }+ err = c.handleRateLimits(ctx, resp)+ if err != nil {+ return nil, err}body, err := io.ReadAll(resp.Body)@@ -315,6 +305,60 @@ func (c *Client) getPostmanResponseBodyBytes(ctx trContext.Context, urlString streturn body, nil}+// handleRateLimits processes the rate limit headers from the Postman API response+// and updates the client's metrics accordingly. If the monthly rate limit usage exceeds+// the set threshold, an error is returned to abort further processing.+func (c *Client) handleRateLimits(ctx trContext.Context, resp *http.Response) error {+ rateLimitRemainingMonthValue := resp.Header.Get("RateLimit-Remaining-Month")+ if rateLimitRemainingMonthValue == "" {+ rateLimitRemainingMonthValue = resp.Header.Get("X-RateLimit-Remaining-Month")+ }+ if rateLimitRemainingMonthValue == "" {+ ctx.Logger().V(2).Info("RateLimit-Remaining-Month header not found in response")+ return nil+ }++ rateLimitRemainingMonth, err := strconv.Atoi(rateLimitRemainingMonthValue)+ if err != nil {+ ctx.Logger().Error(err, "Couldn't convert RateLimit-Remaining-Month to an int",+ "header_value", rateLimitRemainingMonthValue,+ )+ return nil+ }+ c.Metrics.apiMonthlyRequestsRemaining.WithLabelValues().Set(+ float64(rateLimitRemainingMonth),+ )++ rateLimitTotalMonthValue := resp.Header.Get("RateLimit-Limit-Month")+ if rateLimitTotalMonthValue == "" {+ rateLimitTotalMonthValue = resp.Header.Get("X-RateLimit-Limit-Month")+ }+ if rateLimitTotalMonthValue == "" {+ ctx.Logger().V(2).Info("RateLimit-Limit-Month header not found in response")+ return nil+ }+ rateLimitTotalMonth, err := strconv.Atoi(rateLimitTotalMonthValue)+ if err != nil {+ ctx.Logger().Error(err, "Couldn't convert RateLimit-Limit-Month to an int",+ "header_value", rateLimitTotalMonthValue,+ )+ return nil+ }++ if rateLimitTotalMonth == 0 {+ ctx.Logger().V(2).Info("RateLimit-Limit-Month is zero, cannot compute usage percentage")+ return nil+ }++ // failsafe to abandon scan if we are over the threshold of monthly API requests used+ percentageUsed := float64(rateLimitTotalMonth-rateLimitRemainingMonth) / float64(rateLimitTotalMonth)+ if percentageUsed > abortScanAPIReqLimitThreshold {+ return errAbortScanDueToAPIRateLimit+ }++ return nil+}+// EnumerateWorkspaces returns the workspaces for a given user (both private, public, team and personal).// Consider adding additional flags to support filtering.func (c *Client) EnumerateWorkspaces(ctx trContext.Context) ([]Workspace, error) {diff --git a/pkg/sources/postman/postman_test.go b/pkg/sources/postman/postman_test.goindex c896e6d36..1bb4863bb 100644--- a/pkg/sources/postman/postman_test.go+++ b/pkg/sources/postman/postman_test.go@@ -8,6 +8,7 @@ import ("testing""time"+ "github.com/go-errors/errors""github.com/stretchr/testify/assert""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -734,3 +735,67 @@ func TestSource_ResponseID(t *testing.T) {}}++func TestSource_AbortScanRateLimitCrossedThreshold(t *testing.T) {+ defer gock.Off()+ // Mock the API response for getting collection+ gock.New("https://api.getpostman.com").+ Get("/collections/1234-abc1").+ Reply(200).+ AddHeader("X-RateLimit-Remaining-Month", "19").+ AddHeader("X-RateLimit-Limit-Month", "100").+ BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",+ "updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},+ "item":[]}}`)++ ctx := context.Background()+ s, conn := createTestSource(&sourcespb.Postman{+ Credential: &sourcespb.Postman_Token{+ Token: "super-secret-token",+ },+ })++ err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)+ if err != nil {+ t.Fatalf("init error: %v", err)+ }+ gock.InterceptClient(s.client.HTTPClient)+ defer gock.RestoreClient(s.client.HTTPClient)++ _, err = s.client.GetCollection(ctx, "1234-abc1")+ if !errors.Is(err, errAbortScanDueToAPIRateLimit) {+ t.Errorf("expected abort scan error due to rate limit, got: %v", err)+ }+}++func TestSource_AbortScanRateLimitBelowThreshold(t *testing.T) {+ defer gock.Off()+ // Mock the API response for getting collection+ gock.New("https://api.getpostman.com").+ Get("/collections/1234-abc1").+ Reply(200).+ AddHeader("X-RateLimit-Remaining-Month", "90").+ AddHeader("X-RateLimit-Limit-Month", "100").+ BodyString(`{"collection":{"info":{"_postman_id":"abc1","name":"test-collection-1","schema":"https://schema.postman.com/json/collection/v2.1.0/collection.json",+ "updatedAt":"2025-03-21T17:39:25.000Z","createdAt":"2025-03-21T17:37:13.000Z","lastUpdatedBy":"1234","uid":"1234-abc1"},+ "item":[]}}`)++ ctx := context.Background()+ s, conn := createTestSource(&sourcespb.Postman{+ Credential: &sourcespb.Postman_Token{+ Token: "super-secret-token",+ },+ })++ err := s.Init(ctx, "test - postman", 0, 1, false, conn, 1)+ if err != nil {+ t.Fatalf("init error: %v", err)+ }+ gock.InterceptClient(s.client.HTTPClient)+ defer gock.RestoreClient(s.client.HTTPClient)++ _, err = s.client.GetCollection(ctx, "1234-abc1")+ if err != nil {+ t.Errorf("expected no error, got: %v", err)+ }+}