Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Simplified Gitlab Enumeration (#4283)
Kashif Khan7907482dec76
diff --git a/main.go b/main.goindex 19863b27d..553b4b398 100644--- a/main.go+++ b/main.go@@ -450,6 +450,9 @@ func run(state overseer.State) {// OSS Default APK handling onfeature.EnableAPKHandler.Store(true)+ // OSS Default simplified gitlab enumeration+ feature.UseSimplifiedGitlabEnumeration.Store(true)+conf := &config.Config{}if *configFilename != "" {var err errordiff --git a/pkg/feature/feature.go b/pkg/feature/feature.goindex db8036f5d..b329c1d44 100644--- a/pkg/feature/feature.go+++ b/pkg/feature/feature.go@@ -3,11 +3,12 @@ package featureimport "sync/atomic"var (- ForceSkipBinaries atomic.Bool- ForceSkipArchives atomic.Bool- SkipAdditionalRefs atomic.Bool- EnableAPKHandler atomic.Bool- UserAgentSuffix AtomicString+ ForceSkipBinaries atomic.Bool+ ForceSkipArchives atomic.Bool+ SkipAdditionalRefs atomic.Bool+ EnableAPKHandler atomic.Bool+ UserAgentSuffix AtomicString+ UseSimplifiedGitlabEnumeration atomic.Bool)type AtomicString struct {diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 44e2edc3b..7614ccbe5 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -10,6 +10,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/feature""github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/log""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"@@ -265,9 +266,16 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tarreturn ctx.Err()},}- if err := s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter); err != nil {- return err+ if feature.UseSimplifiedGitlabEnumeration.Load() {+ if err := s.getAllProjectReposV2(ctx, apiClient, ignoreRepo, reporter); err != nil {+ return err+ }+ } else {+ if err := s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter); err != nil {+ return err+ }}+} else {gitlabReposEnumerated.WithLabelValues(s.name).Set(float64(len(repos)))}@@ -392,9 +400,17 @@ func (s *Source) Validate(ctx context.Context) []error {return nil},}- if err := s.getAllProjectRepos(ctx, apiClient, ignoreProject, visitor); err != nil {- errs = append(errs, err)- return errs++ if feature.UseSimplifiedGitlabEnumeration.Load() {+ if err := s.getAllProjectReposV2(ctx, apiClient, ignoreProject, visitor); err != nil {+ errs = append(errs, err)+ return errs+ }+ } else {+ if err := s.getAllProjectRepos(ctx, apiClient, ignoreProject, visitor); err != nil {+ errs = append(errs, err)+ return errs+ }}if len(repos) == 0 {@@ -453,9 +469,8 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {return false}-// getAllProjectRepos enumerates all GitLab projects using the provided API-// client. The reporter is used to report the valid repository found for-// projects that are not ignored.+// getAllProjectRepos enumerates all GitLab projects using the provided API client.+// The reporter is used to report the valid repository found for projects that are not ignored.func (s *Source) getAllProjectRepos(ctx context.Context,apiClient *gitlab.Client,@@ -616,6 +631,106 @@ func (s *Source) getAllProjectRepos(return nil}+// getAllProjectReposV2 uses simplified logic to enumerate through all projects using list-all-projects API.+// The reporter is used to report the valid repository found for projects that are not ignored.+func (s *Source) getAllProjectReposV2(+ ctx context.Context,+ apiClient *gitlab.Client,+ ignoreRepo func(string) bool,+ reporter sources.UnitReporter,+) error {+ gitlabReposEnumerated.WithLabelValues(s.name).Set(0)++ // record the projectsWithNamespace for logging.+ var projectsWithNamespace []string++ const (+ orderBy = "id" // TODO: use keyset pagination (https://docs.gitlab.com/ee/api/rest/index.html#keyset-based-pagination)+ paginationLimit = 100 // default is 20, max is 100.+ )++ listOpts := gitlab.ListOptions{PerPage: paginationLimit}+ projectQueryOptions := &gitlab.ListProjectsOptions{+ OrderBy: gitlab.Ptr(orderBy),+ ListOptions: listOpts,+ Membership: gitlab.Ptr(true),+ }++ // for non gitlab.com instances, include all available projects (public + membership).+ if s.url != gitlabBaseURL {+ projectQueryOptions.Membership = gitlab.Ptr(false)+ }++ ctx.Logger().Info("starting projects enumeration",+ "list_options", listOpts,+ "all_available", *projectQueryOptions.Membership)++ // paginate through all projects until no more pages remain.+ for {+ projects, res, err := apiClient.Projects.ListProjects(projectQueryOptions)+ if err != nil {+ err = fmt.Errorf("received error on listing projects: %w", err)+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }++ break+ }++ ctx.Logger().V(3).Info("listed projects", "count", len(projects))++ // process each project+ for _, proj := range projects {+ projCtx := context.WithValues(ctx,+ "project_id", proj.ID,+ "project_name", proj.NameWithNamespace)++ // skip projects configured to be ignored.+ if ignoreRepo(proj.PathWithNamespace) {+ projCtx.Logger().V(3).Info("skipping project", "reason", "ignored in config")++ continue+ }++ // report an error if we could not convert the project into a URL.+ if _, err := url.Parse(proj.HTTPURLToRepo); err != nil {+ projCtx.Logger().V(3).Info("skipping project",+ "reason", "URL parse failure",+ "url", proj.HTTPURLToRepo,+ "parse_error", err)++ err = fmt.Errorf("could not parse url %q given by project: %w", proj.HTTPURLToRepo, err)+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }++ continue+ }++ // report the unit.+ projCtx.Logger().V(3).Info("accepting project")++ unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo}+ gitlabReposEnumerated.WithLabelValues(s.name).Inc()+ projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)++ if err := reporter.UnitOk(ctx, unit); err != nil {+ return err+ }+ }++ // handle pagination.+ projectQueryOptions.Page = res.NextPage+ if res.NextPage == 0 {+ break+ }+ }++ ctx.Logger().Info("Enumerated GitLab projects", "count", len(projectsWithNamespace))++ return nil+}+func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {// If there is resume information available, limit this scan to only the repos that still need scanning.reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)@@ -824,7 +939,12 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e// TODO: Handle error returned from UnitErr._ = reporter.UnitErr(ctx, fmt.Errorf("could not compile include/exclude repo glob: %w", err))})- return s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter)++ if feature.UseSimplifiedGitlabEnumeration.Load() {+ return s.getAllProjectReposV2(ctx, apiClient, ignoreRepo, reporter)+ } else {+ return s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter)+ }}// ChunkUnit downloads and reports chunks for the given GitLab repository unit.