Skip to content

Find, verify, and analyze leaked credentials

Added feature flag to configure projects per page in gitlab enumeration (#4437)

Kashif Khan18c7b1fc33e6
diff --git a/main.go b/main.go
index 04d372f33..cbb263bd2 100644
--- a/main.go
+++ b/main.go
@@ -463,6 +463,7 @@ func run(state overseer.State) {
// OSS Default simplified gitlab enumeration
feature.UseSimplifiedGitlabEnumeration.Store(true)
+ feature.GitlabProjectsPerPage.Store(100)
conf := &config.Config{}
if *configFilename != "" {
diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go
index dab05c293..810057058 100644
--- a/pkg/engine/gitlab.go
+++ b/pkg/engine/gitlab.go
@@ -63,6 +63,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (source
}
connection.NoCleanup = c.NoCleanup
+
connection.PrintLegacyJson = c.PrintLegacyJSON
var conn anypb.Any
diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go
index 53a67230c..3c030a631 100644
--- a/pkg/feature/feature.go
+++ b/pkg/feature/feature.go
@@ -9,7 +9,8 @@ var (
EnableAPKHandler atomic.Bool
UserAgentSuffix AtomicString
UseSimplifiedGitlabEnumeration atomic.Bool
- UseGitMirror atomic.Bool
+ UseGitMirror atomic.Bool
+ GitlabProjectsPerPage atomic.Int64
)
type AtomicString struct {
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index f3f515a13..b8c9b5fe6 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -66,9 +66,12 @@ type Source struct {
useAuthInUrl bool
- clonePath string
- noCleanup bool
+ clonePath string
+ noCleanup bool
+
printLegacyJSON bool
+
+ projectsPerPage int
}
// WithCustomContentWriter sets the useCustomContentWriter flag on the source.
@@ -171,6 +174,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou
s.clonePath = conn.GetClonePath()
s.noCleanup = conn.GetNoCleanup()
s.printLegacyJSON = conn.GetPrintLegacyJson()
+ s.projectsPerPage = int(feature.GitlabProjectsPerPage.Load())
+
+ if s.projectsPerPage > 100 {
+ return fmt.Errorf("invalid config: maximum allowed projects per page for gitlab is 100")
+ }
// configuration uses the inverse logic of the `useAuthInUrl` flag.
s.useAuthInUrl = !conn.RemoveAuthInUrl
@@ -546,10 +554,10 @@ func (s *Source) getAllProjectRepos(
}
const (
- orderBy = "id" // TODO: Use keyset pagination (https://docs.gitlab.com/ee/api/rest/index.html#keyset-based-pagination)
- paginationLimit = 100 // Default is 20, max is 100.
+ orderBy = "id"
)
- listOpts := gitlab.ListOptions{PerPage: paginationLimit}
+ // Trufflehog default per page 100 unless set to other value through feature flag. If 0 provided in feature flag gitlab default it to 20
+ listOpts := gitlab.ListOptions{PerPage: s.projectsPerPage}
projectQueryOptions := &gitlab.ListProjectsOptions{OrderBy: gitlab.Ptr(orderBy), ListOptions: listOpts}
for {
@@ -654,14 +662,13 @@ func (s *Source) getAllProjectReposV2(
) error {
gitlabReposEnumerated.WithLabelValues(s.name).Set(0)
- const paginationLimit = 100 // default is 20, max is 100.
-
// example: https://gitlab.com/gitlab-org/api/client-go/-/blob/main/examples/pagination.go#L55
listOpts := gitlab.ListOptions{
OrderBy: "id",
Pagination: "keyset", // https://docs.gitlab.com/api/rest/#keyset-based-pagination
- PerPage: paginationLimit,
- Sort: "asc",
+ // Trufflehog default per page 100 unless set to other value through feature flag. If 0 provided in feature flag gitlab default it to 20
+ PerPage: s.projectsPerPage,
+ Sort: "asc",
}
projectQueryOptions := &gitlab.ListProjectsOptions{
@@ -756,11 +763,10 @@ func (s *Source) getAllProjectReposInGroups(
var projectsWithNamespace []string
const (
- orderBy = "id"
- paginationLimit = 100
+ orderBy = "id"
)
- listOpts := gitlab.ListOptions{PerPage: paginationLimit}
+ listOpts := gitlab.ListOptions{PerPage: s.projectsPerPage}
projectOpts := &gitlab.ListGroupProjectsOptions{
ListOptions: listOpts,
OrderBy: gitlab.Ptr(orderBy),