Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Log analyze errors for HuggingFace analyzer (#5130)
Kashif Khan4ecd7d56fdb6
diff --git a/pkg/analyzer/analyzers/huggingface/huggingface.go b/pkg/analyzer/analyzers/huggingface/huggingface.goindex 73bf9b1a9..893f988f5 100644--- a/pkg/analyzer/analyzers/huggingface/huggingface.go+++ b/pkg/analyzer/analyzers/huggingface/huggingface.go@@ -5,6 +5,7 @@ package huggingfaceimport ("encoding/json""fmt"+ "io""net/http""os""strings"@@ -30,13 +31,15 @@ type Analyzer struct {func (Analyzer) Type() analyzers.AnalyzerType { return analyzers.AnalyzerTypeHuggingFace }-func (a Analyzer) Analyze(_ context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {+func (a Analyzer) Analyze(ctx context.Context, credInfo map[string]string) (*analyzers.AnalyzerResult, error) {key, ok := credInfo["key"]if !ok || key == "" {- return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", fmt.Errorf("key not found in credentialInfo"))+ err := fmt.Errorf("key not found in credentialInfo")+ ctx.Logger().Error(err, "huggingface credentials missing key")+ return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationValidateCredentials, analyzers.ServiceConfig, "", err)}- info, err := AnalyzePermissions(a.Cfg, key)+ info, err := AnalyzePermissions(ctx, a.Cfg, key)if err != nil {return nil, analyzers.NewAnalysisError(a.Type().String(), analyzers.OperationAnalyzePermissions, analyzers.ServiceAPI, "", err)}@@ -309,13 +312,14 @@ type Model struct {// getModelsByAuthor calls the HF API /models endpoint with the author query param// returns a list of models and an error-func getModelsByAuthor(cfg *config.Config, key string, author string) ([]Model, error) {+func getModelsByAuthor(ctx context.Context, cfg *config.Config, key string, author string) ([]Model, error) {var modelsJSON []Model// create a new requestclient := analyzers.NewAnalyzeClient(cfg)- req, err := http.NewRequest("GET", "https://huggingface.co/api/models", nil)+ req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/models", nil)if err != nil {+ ctx.Logger().Error(err, "failed to create huggingface models request", "author", author)return modelsJSON, err}@@ -330,14 +334,25 @@ func getModelsByAuthor(cfg *config.Config, key string, author string) ([]Model,// send the requestresp, err := client.Do(req)if err != nil {+ ctx.Logger().Error(err, "failed to send huggingface models request", "author", author)return modelsJSON, err}// defer the response body closing- defer func() { _ = resp.Body.Close() }()+ defer func() {+ _, _ = io.Copy(io.Discard, resp.Body)+ _ = resp.Body.Close()+ }()++ if resp.StatusCode != http.StatusOK {+ err := fmt.Errorf("unexpected status code: %d while fetching models for author %s", resp.StatusCode, author)+ ctx.Logger().Error(err, "unexpected status code while fetching huggingface models", "author", author, "status_code", resp.StatusCode)+ return modelsJSON, err+ }// read responseif err := json.NewDecoder(resp.Body).Decode(&modelsJSON); err != nil {+ ctx.Logger().Error(err, "failed to decode huggingface models response", "author", author)return modelsJSON, err}return modelsJSON, nil@@ -345,13 +360,14 @@ func getModelsByAuthor(cfg *config.Config, key string, author string) ([]Model,// getTokenInfo calls the HF API /whoami-v2 endpoint to get the token info// returns the token info, a boolean indicating token validity, and an error-func getTokenInfo(cfg *config.Config, key string) (HFTokenJSON, bool, error) {+func getTokenInfo(ctx context.Context, cfg *config.Config, key string) (HFTokenJSON, bool, error) {var tokenJSON HFTokenJSON// create a new requestclient := analyzers.NewAnalyzeClient(cfg)- req, err := http.NewRequest("GET", "https://huggingface.co/api/whoami-v2", nil)+ req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil)if err != nil {+ ctx.Logger().Error(err, "failed to create huggingface whoami request")return tokenJSON, false, err}@@ -361,19 +377,25 @@ func getTokenInfo(cfg *config.Config, key string) (HFTokenJSON, bool, error) {// send the requestresp, err := client.Do(req)if err != nil {+ ctx.Logger().Error(err, "failed to send huggingface whoami request")return tokenJSON, false, err}+ // defer the response body closing+ defer func() {+ _, _ = io.Copy(io.Discard, resp.Body)+ _ = resp.Body.Close()+ }()+// check if the response is 200- if resp.StatusCode != 200 {+ if resp.StatusCode != http.StatusOK {+ ctx.Logger().V(2).Info("huggingface token invalid while fetching whoami", "status_code", resp.StatusCode)return tokenJSON, false, nil}- // defer the response body closing- defer func() { _ = resp.Body.Close() }()-// read responseif err := json.NewDecoder(resp.Body).Decode(&tokenJSON); err != nil {+ ctx.Logger().Error(err, "failed to decode huggingface whoami response")return tokenJSON, true, err}return tokenJSON, true, nil@@ -384,20 +406,22 @@ type SecretInfo struct {Models []Model}-func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) {+func AnalyzePermissions(ctx context.Context, cfg *config.Config, key string) (*SecretInfo, error) {// get token info- token, success, err := getTokenInfo(cfg, key)+ token, success, err := getTokenInfo(ctx, cfg, key)if err != nil {return nil, err}if !success {- return nil, fmt.Errorf("invalid HuggingFace Access Token")+ err := fmt.Errorf("invalid HuggingFace Access Token")+ ctx.Logger().V(2).Info("huggingface access token validation failed")+ return nil, err}// get all models by usernamevar allModels []Model- userModels, err := getModelsByAuthor(cfg, key, token.Username)+ userModels, err := getModelsByAuthor(ctx, cfg, key, token.Username)if err != nil {return nil, err}@@ -405,7 +429,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) {// get all models from all orgsfor _, org := range token.Orgs {- orgModels, err := getModelsByAuthor(cfg, key, org.Name)+ orgModels, err := getModelsByAuthor(ctx, cfg, key, org.Name)if err != nil {return nil, err}@@ -420,7 +444,7 @@ func AnalyzePermissions(cfg *config.Config, key string) (*SecretInfo, error) {// AnalyzeAndPrintPermissions prints the permissions of a HuggingFace API keyfunc AnalyzeAndPrintPermissions(cfg *config.Config, key string) {- info, err := AnalyzePermissions(cfg, key)+ info, err := AnalyzePermissions(context.Background(), cfg, key)if err != nil {color.Red("[x] Error: %s", err.Error())return