Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Implement SourceUnitEnumChunker for GitLab (#2367)
Miccahaace92b64d85
diff --git a/pkg/engine/gitlab_integration_test.go b/pkg/engine/gitlab_integration_test.gonew file mode 100644index 000000000..dbd45ea43--- /dev/null+++ b/pkg/engine/gitlab_integration_test.go@@ -0,0 +1,41 @@+//go:build integration+// +build integration++package engine++import (+ "fmt"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++func TestGitLab(t *testing.T) {+ // Run the scan.+ ctx := context.Background()+ e, err := Start(ctx,+ WithDetectors(DefaultDetectors()...),+ WithVerify(false),+ )+ assert.NoError(t, err)++ secret, err := common.GetTestSecret(ctx)+ if err != nil {+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ }+ err = e.ScanGitLab(ctx, sources.GitlabConfig{+ Token: secret.MustGetField("GITLAB_TOKEN"),+ })+ assert.NoError(t, err)++ err = e.Finish(ctx)+ assert.NoError(t, err)++ // Check the output provided by metrics.+ metrics := e.GetMetrics()+ assert.GreaterOrEqual(t, metrics.ChunksScanned, uint64(36312))+ assert.GreaterOrEqual(t, metrics.BytesScanned, uint64(91618854))+}diff --git a/pkg/sources/chan_reporter.go b/pkg/sources/chan_reporter.goindex 11c161980..56adad83f 100644--- a/pkg/sources/chan_reporter.go+++ b/pkg/sources/chan_reporter.go@@ -18,5 +18,24 @@ func (c ChanReporter) ChunkOk(ctx context.Context, chunk Chunk) error {func (ChanReporter) ChunkErr(ctx context.Context, err error) error {ctx.Logger().Error(err, "error chunking")- return nil+ return ctx.Err()+}++var _ UnitReporter = (*VisitorReporter)(nil)++type VisitorReporter struct {+ VisitUnit func(context.Context, SourceUnit) error+ VisitErr func(context.Context, error) error+}++func (v VisitorReporter) UnitOk(ctx context.Context, unit SourceUnit) error {+ return v.VisitUnit(ctx, unit)+}++func (v VisitorReporter) UnitErr(ctx context.Context, err error) error {+ if v.VisitErr == nil {+ ctx.Logger().Error(err, "error enumerating")+ return ctx.Err()+ }+ return v.VisitErr(ctx, err)}diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 62ff13db1..850d153b3 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -62,6 +62,7 @@ func (s *Source) WithCustomContentWriter() { s.useCustomContentWriter = true }var _ sources.Source = (*Source)(nil)var _ sources.SourceUnitUnmarshaller = (*Source)(nil)var _ sources.Validator = (*Source)(nil)+var _ sources.SourceUnitEnumChunker = (*Source)(nil)// Type returns the type of source.// It is used for matching source types in configuration and job input.@@ -183,11 +184,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .ignoreRepo := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", pattern)})- gitlabRepos, err := s.getReposFromGitlab(ctx, apiClient, ignoreRepo)- if err != nil {+ reporter := sources.VisitorReporter{+ VisitUnit: func(ctx context.Context, unit sources.SourceUnit) error {+ repos = append(repos, unit.SourceUnitID())+ return ctx.Err()+ },+ }+ if err := s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter); err != nil {return err}- repos = gitlabRepos}s.repos = repos@@ -246,19 +251,24 @@ func (s *Source) Validate(ctx context.Context) []error {errs = append(errs, fmt.Errorf("could not compile ignore repo pattern %q: %w", pattern, err))})- projects, err := s.getAllProjects(ctx, apiClient)- if err != nil {+ // Query GitLab for the list of configured repos.+ var repos []string+ visitor := sources.VisitorReporter{+ VisitUnit: func(ctx context.Context, unit sources.SourceUnit) error {+ repos = append(repos, unit.SourceUnitID())+ return nil+ },+ }+ if err := s.getAllProjectRepos(ctx, apiClient, ignoreProject, visitor); err != nil {errs = append(errs, err)return errs}- for _, p := range projects {- if !ignoreProject(p.PathWithNamespace) {- return errs- }+ if len(repos) == 0 {+ errs = append(errs, fmt.Errorf("ignore patterns excluded all projects"))}- return append(errs, fmt.Errorf("ignore patterns excluded all projects"))+ return errs}func (s *Source) newClient() (*gitlab.Client, error) {@@ -310,28 +320,54 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {return false}-func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) {+// getAllProjectRepos enumerates all GitLab projects using the provided API+// client. The reporter is used to report the valid repository found for+// projects that are not ignored.+func (s *Source) getAllProjectRepos(+ ctx context.Context,+ apiClient *gitlab.Client,+ ignoreRepo func(string) bool,+ reporter sources.UnitReporter,+) error {// Projects without repo will get user projects, groups projects, and subgroup projects.user, _, err := apiClient.Users.CurrentUser()if err != nil {- return nil, fmt.Errorf("unable to authenticate using %s: %w", s.authMethod, err)+ return fmt.Errorf("unable to authenticate using %s: %w", s.authMethod, err)}uniqueProjects := make(map[int]*gitlab.Project)- var (- projects []*gitlab.Project- projectsWithNamespace []string- )+ // Record the projectsWithNamespace for logging.+ var projectsWithNamespace []string// Used to filter out duplicate projects.- processProjects := func(projList []*gitlab.Project) {+ processProjects := func(projList []*gitlab.Project) error {for _, proj := range projList {- if _, exists := uniqueProjects[proj.ID]; !exists {- uniqueProjects[proj.ID] = proj- projects = append(projects, proj)- projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)+ // Skip projects we've already seen.+ if _, exists := uniqueProjects[proj.ID]; exists {+ continue+ }+ // Skip projects configured to be ignored.+ if ignoreRepo(proj.PathWithNamespace) {+ continue+ }+ // Record that we've seen this project.+ uniqueProjects[proj.ID] = proj+ projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)+ // Report an error if we could not convert the project into a URL.+ if _, err := url.Parse(proj.HTTPURLToRepo); err != nil {+ err = fmt.Errorf("could not parse url %q given by project: %w", proj.HTTPURLToRepo, err)+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }+ continue+ }+ // Report the unit.+ unit := sources.CommonSourceUnit{ID: proj.HTTPURLToRepo}+ if err := reporter.UnitOk(ctx, unit); err != nil {+ return err}}+ return nil}const (@@ -344,9 +380,15 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (for {userProjects, res, err := apiClient.Projects.ListUserProjects(user.ID, projectQueryOptions)if err != nil {- return nil, fmt.Errorf("received error on listing user projects: %w", err)+ err = fmt.Errorf("received error on listing user projects: %w", err)+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }+ break+ }+ if err := processProjects(userProjects); err != nil {+ return err}- processProjects(userProjects)projectQueryOptions.Page = res.NextPageif res.NextPage == 0 {break@@ -368,7 +410,11 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (for {groupList, res, err := apiClient.Groups.ListGroups(&listGroupsOptions)if err != nil {- return nil, fmt.Errorf("received error on listing groups, you probably don't have permissions to do that: %w", err)+ err = fmt.Errorf("received error on listing groups, you probably don't have permissions to do that: %w", err)+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }+ break}groups = append(groups, groupList...)listGroupsOptions.Page = res.NextPage@@ -386,13 +432,18 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (for {grpPrjs, res, err := apiClient.Groups.ListGroupProjects(group.ID, listGroupProjectOptions)if err != nil {- ctx.Logger().Info("received error on listing group projects, you probably don't have permissions to do that",- "group", group.FullPath,- "error", err,+ err = fmt.Errorf(+ "received error on listing group projects for %q, you probably don't have permissions to do that: %w",+ group.FullPath, err,)+ if err := reporter.UnitErr(ctx, err); err != nil {+ return err+ }break}- processProjects(grpPrjs)+ if err := processProjects(grpPrjs); err != nil {+ return err+ }listGroupProjectOptions.Page = res.NextPageif res.NextPage == 0 {break@@ -400,38 +451,10 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (}}- ctx.Logger().Info("Enumerated GitLab projects", "count", len(projects))+ ctx.Logger().Info("Enumerated GitLab projects", "count", len(projectsWithNamespace))ctx.Logger().V(2).Info("Enumerated GitLab projects", "projects", projectsWithNamespace)- return projects, nil-}--func (s *Source) getReposFromGitlab(ctx context.Context, apiClient *gitlab.Client, ignoreRepo func(repo string) bool) ([]string, error) {- projects, err := s.getAllProjects(ctx, apiClient)- if err != nil {- return nil, fmt.Errorf("error getting all projects: %w", err)- }-- // Turn projects into URLs for Git cloner.- var repos []string- for _, prj := range projects {- if ignoreRepo(prj.PathWithNamespace) {- continue- }-- // Ensure the urls are valid before adding them to the repo list.- _, err := url.Parse(prj.HTTPURLToRepo)- if err != nil {- ctx.Logger().Error(err, "could not parse url given by project", "project", prj.HTTPURLToRepo)- continue- }- repos = append(repos, prj.HTTPURLToRepo)- }- if len(repos) == 0 {- return nil, fmt.Errorf("unable to discover any repos")- }-- return repos, nil+ return nil}func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {@@ -564,3 +587,74 @@ func normalizeRepos(repos []string) ([]string, []error) {}return validRepos, errs}++// Enumerate reports all GitLab repositories to be scanned to the reporter. If+// none are configured, it will find all repositories within all projects that+// the configured user has access to, while respecting the configured ignore+// rules.+func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {+ // Start client.+ apiClient, err := s.newClient()+ if err != nil {+ return err+ }++ // Get repos within target.+ repos, errs := normalizeRepos(s.repos)+ for _, repoErr := range errs {+ ctx.Logger().Info("error normalizing repo", "error", repoErr)+ if err := reporter.UnitErr(ctx, repoErr); err != nil {+ return err+ }+ }++ // End early if we had errors getting specified repos but none were validated.+ if len(errs) > 0 && len(repos) == 0 {+ return fmt.Errorf("all configured repos had validation issues")+ }++ // Report all repos if specified.+ if len(repos) > 0 {+ for _, repo := range repos {+ unit := sources.CommonSourceUnit{ID: repo}+ if err := reporter.UnitOk(ctx, unit); err != nil {+ return err+ }+ }+ return nil+ }++ // Otherwise, enumerate all repos.+ ignoreRepo := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {+ ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", pattern)+ // TODO: Handle error returned from UnitErr.+ _ = reporter.UnitErr(ctx, fmt.Errorf("could not compile ignore repo glob: %w", err))+ })+ return s.getAllProjectRepos(ctx, apiClient, ignoreRepo, reporter)+}++// ChunkUnit downloads and reports chunks for the given GitLab repository unit.+func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {+ repoURL := unit.SourceUnitID()++ var path string+ var repo *gogit.Repository+ var err error+ if s.authMethod == "UNAUTHENTICATED" {+ path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL)+ } else {+ // If a username is not provided we need to use a default one in order to clone a private repo.+ // Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.+ user := s.user+ if user == "" {+ user = "placeholder"+ }+ path, repo, err = git.CloneRepoUsingToken(ctx, s.token, repoURL, user)+ }+ if err != nil {+ return err+ }+ defer os.RemoveAll(path)++ return s.git.ScanRepo(ctx, repo, path, s.scanOptions, reporter)+}