Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Removed stored logger from HuggingFace source (#4328)
Kashif Khandbfe811b48cd
diff --git a/pkg/sources/huggingface/client.go b/pkg/sources/huggingface/client.goindex 3d3aa72d3..d23626580 100644--- a/pkg/sources/huggingface/client.go+++ b/pkg/sources/huggingface/client.go@@ -150,11 +150,11 @@ func (c *HFClient) get(ctx context.Context, url string, target interface{}) erro}if resp.StatusCode == http.StatusUnauthorized {- return errors.New("invalid API key.")+ return errors.New("invalid API key")}if resp.StatusCode == http.StatusForbidden {- return errors.New("access to this repo is restricted and you are not in the authorized list. Visit the repository to ask for access.")+ return errors.New("access to this repo is restricted and you are not in the authorized list. Visit the repository to ask for access")}defer resp.Body.Close()diff --git a/pkg/sources/huggingface/huggingface.go b/pkg/sources/huggingface/huggingface.goindex 462f33df6..fb3abdc05 100644--- a/pkg/sources/huggingface/huggingface.go+++ b/pkg/sources/huggingface/huggingface.go@@ -68,7 +68,6 @@ type Source struct {scanOptions *git.ScanOptionsapiClient *HFClient- log logr.Loggerconn *sourcespb.HuggingfacejobPool *errgroup.GroupresumeInfoMutex sync.Mutex@@ -112,13 +111,13 @@ type filteredRepoCache struct {include, exclude []glob.Glob}-func (s *Source) newFilteredRepoCache(c cache.Cache[string], include, exclude []string) *filteredRepoCache {+func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string], include, exclude []string) *filteredRepoCache {includeGlobs := make([]glob.Glob, 0, len(include))excludeGlobs := make([]glob.Glob, 0, len(exclude))for _, ig := range include {g, err := glob.Compile(ig)if err != nil {- s.log.V(1).Info("invalid include glob", "include_value", ig, "err", err)+ ctx.Logger().V(1).Info("invalid include glob", "include_value", ig, "err", err)continue}includeGlobs = append(includeGlobs, g)@@ -126,7 +125,7 @@ func (s *Source) newFilteredRepoCache(c cache.Cache[string], include, exclude []for _, eg := range exclude {g, err := glob.Compile(eg)if err != nil {- s.log.V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)+ ctx.Logger().V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)continue}excludeGlobs = append(excludeGlobs, g)@@ -169,14 +168,12 @@ func (c *filteredRepoCache) includeRepo(s string) bool {}// Init returns an initialized HuggingFace source.-func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {+func (s *Source) Init(ctx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {err := git.CmdCheck()if err != nil {return err}- s.log = aCtx.Logger()-s.name = names.sourceID = sourceIDs.jobID = jobID@@ -208,17 +205,17 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, soreturn err}- s.filteredModelsCache = s.newFilteredRepoCache(simple.NewCache[string](),+ s.filteredModelsCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](),append(s.conn.GetModels(), s.conn.GetIncludeModels()...),s.conn.GetIgnoreModels(),)- s.filteredSpacesCache = s.newFilteredRepoCache(simple.NewCache[string](),+ s.filteredSpacesCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](),append(s.conn.GetSpaces(), s.conn.GetIncludeSpaces()...),s.conn.GetIgnoreSpaces(),)- s.filteredDatasetsCache = s.newFilteredRepoCache(simple.NewCache[string](),+ s.filteredDatasetsCache = s.newFilteredRepoCache(ctx, simple.NewCache[string](),append(s.conn.GetDatasets(), s.conn.GetIncludeDatasets()...),s.conn.GetIgnoreDatasets(),)@@ -249,8 +246,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, soLink: giturl.GenerateLink(repository, commit, file, line),Timestamp: sanitizer.UTF8(timestamp),Line: line,- Visibility: s.visibilityOf(aCtx, repository),- ResourceType: s.getResourceType(aCtx, repository),+ Visibility: s.visibilityOf(ctx, repository),+ ResourceType: s.getResourceType(ctx, repository),},},}@@ -369,7 +366,7 @@ func (s *Source) enumerate(ctx context.Context) error {}}- s.log.Info("Completed enumeration", "num_models", len(s.models), "num_spaces", len(s.spaces), "num_datasets", len(s.datasets))+ ctx.Logger().Info("Completed enumeration", "num_models", len(s.models), "num_spaces", len(s.spaces), "num_datasets", len(s.datasets))// We must sort the repos so we can resume later if necessary.sort.Strings(s.models)@@ -507,7 +504,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,repos := s.getReposListByType(resourceType)- s.log.V(2).Info("Found "+resourceType+" to scan", "count", len(repos))+ ctx.Logger().V(2).Info("Found "+resourceType+" to scan", "count", len(repos))// If there is resume information available, limit this scan to only the repos that still need scanning.reposToScan, progressIndexOffset := sources.FilterReposToResume(repos, s.GetProgress().EncodedResumeInfo)@@ -539,7 +536,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,if !ok {// This should never happen.err := fmt.Errorf("no repoInfo for URL: %s", repoURL)- s.log.Error(err, "failed to scan "+resourceType)+ ctx.Logger().Error(err, "failed to scan "+resourceType)return nil}repoCtx := context.WithValues(ctx, resourceType, repoURL)@@ -565,7 +562,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk,_ = s.jobPool.Wait()if scanErrs.Count() > 0 {- s.log.V(0).Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs.String())+ ctx.Logger().V(0).Info("failed to scan some repositories", "error_count", scanErrs.Count(), "errors", scanErrs.String())}s.SetProgressComplete(len(repos), len(repos), "Completed HuggingFace "+resourceType+" scan", "")return nildiff --git a/pkg/sources/huggingface/repo.go b/pkg/sources/huggingface/repo.goindex c3e533c45..8e33e641e 100644--- a/pkg/sources/huggingface/repo.go+++ b/pkg/sources/huggingface/repo.go@@ -56,11 +56,13 @@ func (s *Source) cloneRepo(switch s.conn.GetCredential().(type) {case *sourcespb.Huggingface_Unauthenticated:+ ctx.Logger().V(2).Info("cloning repo without authentication", "repo_url", repoURL)path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)if err != nil {return "", nil, err}case *sourcespb.Huggingface_Token:+ ctx.Logger().V(2).Info("cloning repo with token authentication", "repo_url", repoURL)path, repo, err = git.CloneRepoUsingToken(ctx, s.huggingfaceToken, repoURL, "", true)if err != nil {return "", nil, err