Skip to content

Find, verify, and analyze leaked credentials

[chore] - Increase pagination limit (#2154)

ahrav37d9e5eedfdb
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 6923d63f0..bfa565388 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -293,45 +293,62 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) ([]*gitlab.Project, error) {
// Projects without repo will get user projects, groups projects, and subgroup projects.
user, _, err := apiClient.Users.CurrentUser()
-
if err != nil {
- msg := fmt.Sprintf("unable to authenticate using: %s", s.authMethod)
- return nil, errors.WrapPrefix(err, msg, 0)
+ return nil, fmt.Errorf("unable to authenticate using %s, %w", s.authMethod, err)
+ }
+
+ uniqueProjects := make(map[int]*gitlab.Project)
+ var (
+ projects []*gitlab.Project
+ projectsWithNamespace []string
+ )
+
+ // Used to filter out duplicate projects.
+ processProjects := func(projList []*gitlab.Project) {
+ for _, proj := range projList {
+ if _, exists := uniqueProjects[proj.ID]; !exists {
+ uniqueProjects[proj.ID] = proj
+ projects = append(projects, proj)
+ projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)
+ }
+ }
}
- projects := map[int]*gitlab.Project{}
+ const (
+ orderBy = "last_activity_at"
+ paginationLimit = 100 // Default is 20, max is 100.
+ )
+ listOpts := gitlab.ListOptions{PerPage: paginationLimit}
- projectQueryOptions := &gitlab.ListProjectsOptions{
- OrderBy: gitlab.String("last_activity_at"),
- }
+ projectQueryOptions := &gitlab.ListProjectsOptions{OrderBy: gitlab.String(orderBy), ListOptions: listOpts}
for {
userProjects, res, err := apiClient.Projects.ListUserProjects(user.ID, projectQueryOptions)
if err != nil {
- return nil, errors.Errorf("received error on listing user projects: %s\n", err)
- }
- for _, prj := range userProjects {
- projects[prj.ID] = prj
+ return nil, fmt.Errorf("received error on listing user projects: %w", err)
}
+ processProjects(userProjects)
projectQueryOptions.Page = res.NextPage
if res.NextPage == 0 {
break
}
}
- var groups []*gitlab.Group
-
listGroupsOptions := gitlab.ListGroupsOptions{
+ ListOptions: listOpts,
AllAvailable: gitlab.Bool(false), // This actually grabs public groups on public GitLab if set to true.
TopLevelOnly: gitlab.Bool(false),
Owned: gitlab.Bool(false),
}
- if s.url != "https://gitlab.com/" {
+ const cloudBaseURL = "https://gitlab.com/"
+ if s.url != cloudBaseURL {
listGroupsOptions.AllAvailable = gitlab.Bool(true)
}
+
+ var groups []*gitlab.Group
for {
groupList, res, err := apiClient.Groups.ListGroups(&listGroupsOptions)
if err != nil {
- return nil, errors.Errorf("received error on listing groups, you probably don't have permissions to do that: %s\n", err)
+ return nil, fmt.Errorf("received error on listing groups, you probably don't have permissions to do that: %w", err)
}
groups = append(groups, groupList...)
listGroupsOptions.Page = res.NextPage
@@ -342,7 +359,8 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (
for _, group := range groups {
listGroupProjectOptions := &gitlab.ListGroupProjectsOptions{
- OrderBy: gitlab.String("last_activity_at"),
+ ListOptions: listOpts,
+ OrderBy: gitlab.String(orderBy),
IncludeSubGroups: gitlab.Bool(true),
}
for {
@@ -354,26 +372,18 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (
)
break
}
- for _, prj := range grpPrjs {
- projects[prj.ID] = prj
- }
+ processProjects(grpPrjs)
listGroupProjectOptions.Page = res.NextPage
if res.NextPage == 0 {
break
}
}
}
- var projectNamesWithNamespace []string
- for _, project := range projects {
- projectNamesWithNamespace = append(projectNamesWithNamespace, project.NameWithNamespace)
- }
- ctx.Logger().V(2).Info("Enumerated GitLab projects", "count", len(projects), "projects", projectNamesWithNamespace)
- var projectList []*gitlab.Project
- for _, project := range projects {
- projectList = append(projectList, project)
- }
- return projectList, nil
+ ctx.Logger().Info("Enumerated GitLab projects", "count", len(projects))
+ ctx.Logger().V(2).Info("Enumerated GitLab projects", "projects", projectsWithNamespace)
+
+ return projects, nil
}
func (s *Source) getReposFromGitlab(ctx context.Context, apiClient *gitlab.Client, ignoreRepo func(repo string) bool) ([]string, error, bool) {