Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fix GitLab project metadata cache and switch to LRU (#4727)
Kashif Khan7c724e56cb78
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 733792a99..d4a531376 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -1,6 +1,7 @@package gitlabimport (+ "errors""fmt""net/url""os"@@ -10,6 +11,15 @@ import ("sync""time"+ gogit "github.com/go-git/go-git/v5"+ "github.com/gobwas/glob"+ "github.com/hashicorp/golang-lru/v2/expirable"+ gitlab "gitlab.com/gitlab-org/api/client-go"+ "golang.org/x/oauth2"+ "golang.org/x/sync/errgroup"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/feature"@@ -20,20 +30,12 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"-- gogit "github.com/go-git/go-git/v5"- "github.com/gobwas/glob"- gitlab "gitlab.com/gitlab-org/api/client-go"- "golang.org/x/oauth2"- "golang.org/x/sync/errgroup"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/types/known/anypb")const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB-// This is the URL for gitlab hosted at gitlab.com-const gitlabBaseURL = "https://gitlab.com/"+// Base URL for GitLab Cloud (hosted at gitlab.com)+const gitlabCloudBaseURL = "https://gitlab.com/"type Source struct {name string@@ -72,16 +74,22 @@ type Source struct {sources.CommonSourceUnitUnmarshalleruseAuthInUrl bool-- clonePath string- noCleanup bool+ clonePath string+ noCleanup boolprintLegacyJSON boolprojectsPerPage int64// cache of repo URL to project info, used when generating metadata for chunks- repoToProjCache repoToProjectCache+ projectMetadataCache *expirable.LRU[string, *projectMetadata]+}++// projectMetadata represents GitLab project metadata.+type projectMetadata struct {+ id int64+ name string+ owner string}// WithCustomContentWriter sets the useCustomContentWriter flag on the source.@@ -242,11 +250,23 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, souTimestamp: sanitizer.UTF8(info.Timestamp),Line: info.Line,}- proj, ok := s.repoToProjCache.get(info.Repository)+ // check for project metadata in the cache+ project, ok := s.projectMetadataCache.Get(info.Repository)+ if !ok {+ // Long-running scans of a single large repo can outlive the+ // cache TTL between the initial ensureProjectInCache call and+ // later chunks from that same repo. Repopulate on miss.+ s.ensureProjectInCache(ctx, info.Repository)+ project, ok = s.projectMetadataCache.Get(info.Repository)+ }if ok {- gitlabMetadata.ProjectId = int64(proj.id)- gitlabMetadata.ProjectName = proj.name- gitlabMetadata.ProjectOwner = proj.owner+ ctx.Logger().V(5).Info("cache hit inside source metadata func: found project metadata in the cache", "cache_key", info.Repository)+ gitlabMetadata.ProjectId = project.id+ gitlabMetadata.ProjectName = project.name+ gitlabMetadata.ProjectOwner = project.owner+ } else {+ ctx.Logger().Error(errors.New("failed to get repo metadata from cache"),+ "cache miss: not found project metadata in the cache", "cache_key", info.Repository)}return &source_metadatapb.MetaData{@@ -260,9 +280,11 @@ func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sou}s.git = git.NewGit(cfg)- s.repoToProjCache = repoToProjectCache{- cache: make(map[string]*project),- }+ s.projectMetadataCache = expirable.NewLRU[string, *projectMetadata](+ 15000, // upto 15000 entries+ nil,+ 60*time.Minute, // time-based expiration - 1 hour+ )return nil}@@ -314,8 +336,10 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar} else {gitlabReposEnumerated.WithLabelValues(s.name).Set(float64(len(repos)))- // ensure project details for specified repos are cached- // this is required to populate metadata during chunking+ // Ensure project details for the specified repositories are cached.+ // This is required to populate metadata during chunking.+ // Note: Repository URLs are already normalized, so the cache check+ // uses the normalized repo URL directly.for _, repo := range repos {s.ensureProjectInCache(ctx, repo)}@@ -586,7 +610,10 @@ func (s *Source) getAllProjectRepos(}// Report the unit.ctx.Logger().V(3).Info("accepting project")- s.cacheGitlabProject(proj)++ // Cache the GitLab project metadata.+ s.cacheGitlabProjectMetadata(ctx, proj)+unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo}gitlabReposEnumerated.WithLabelValues(s.name).Inc()projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)@@ -630,7 +657,7 @@ func (s *Source) getAllProjectRepos(Owned: gitlab.Ptr(false),}- if s.url != gitlabBaseURL {+ if s.url != gitlabCloudBaseURL {listGroupsOptions.AllAvailable = gitlab.Ptr(true)}@@ -722,7 +749,7 @@ func (s *Source) getAllProjectReposV2(}// for gitlab.com instance, include only projects where the user is a member.- if s.url == gitlabBaseURL {+ if s.url == gitlabCloudBaseURL {projectQueryOptions.Membership = gitlab.Ptr(true)}@@ -783,7 +810,9 @@ func (s *Source) getAllProjectReposV2(// report the unit.projCtx.Logger().V(3).Info("accepting project")- s.cacheGitlabProject(project)+ // Cache the GitLab project metadata.+ s.cacheGitlabProjectMetadata(projCtx, project)+unit := git.SourceUnit{Kind: git.UnitRepo, ID: project.HTTPURLToRepo}gitlabReposEnumerated.WithLabelValues(s.name).Inc()@@ -837,7 +866,7 @@ func (s *Source) getAllProjectReposInGroups(}// For non gitlab.com instances, you might want to adjust access levels- if s.url != gitlabBaseURL {+ if s.url != gitlabCloudBaseURL {projectOpts.MinAccessLevel = gitlab.Ptr(gitlab.GuestPermissions)}@@ -899,7 +928,9 @@ func (s *Source) getAllProjectReposInGroups(// report the unit.projCtx.Logger().V(3).Info("accepting project")- s.cacheGitlabProject(proj)+ // Cache the GitLab project metadata.+ s.cacheGitlabProjectMetadata(projCtx, proj)+unit := git.SourceUnit{Kind: git.UnitRepo, ID: proj.HTTPURLToRepo}gitlabReposEnumerated.WithLabelValues(s.name).Inc()projectsWithNamespace = append(projectsWithNamespace, proj.NameWithNamespace)@@ -981,6 +1012,11 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)}}+ // Re-populate the cache if the entry expired since it was first+ // cached during enumeration; scanRepos can run for hours on+ // large orgs, longer than the cache's TTL.+ s.ensureProjectInCache(ctx, repoURL)+logger.V(2).Info("starting scan", "num", i+1, "total", len(s.repos))if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, sources.ChanReporter{Ch: chunksChan}); err != nil {scanErrs.Add(err)@@ -1022,77 +1058,6 @@ func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {s.scanOptions = scanOptions}-func buildIgnorer(include, exclude []string, onCompile func(err error, pattern string)) func(repo string) bool {-- // compile and load globRepoFilter- globRepoFilter := newGlobRepoFilter(include, exclude, onCompile)-- f := func(repo string) bool {- if !globRepoFilter.includeRepo(repo) || globRepoFilter.ignoreRepo(repo) {- return true- }- return false- }-- return f-}--func normalizeRepos(repos []string) ([]string, []error) {- // Optimistically allocate space for all valid repositories.- validRepos := make([]string, 0, len(repos))- var errs []error- for _, prj := range repos {- repo, err := giturl.NormalizeGitlabRepo(prj)- if err != nil {- errs = append(errs, fmt.Errorf("unable to normalize gitlab repo url %q: %w", prj, err))- continue- }-- validRepos = append(validRepos, repo)- }- return validRepos, errs-}--// normalizeGitlabEndpoint ensures that if an endpoint is going to gitlab.com, we use https://gitlab.com/ as the endpoint.-// If we see the protocol is http, we error, because this shouldn't be used.-// Otherwise, it ensures we are using https as our protocol, if none was provided.-func normalizeGitlabEndpoint(gitlabEndpoint string) (string, error) {- if gitlabEndpoint == "" {- return gitlabBaseURL, nil- }-- gitlabURL, err := url.Parse(gitlabEndpoint)- if err != nil {- return "", err- }-- // We probably didn't receive a URL with a scheme, which messed up the parsing.- if gitlabURL.Host == "" {- gitlabURL, err = url.Parse("https://" + gitlabEndpoint)- if err != nil {- return "", err- }- }-- // If the host is gitlab.com, this is the cloud version, which has only one valid endpoint.- if gitlabURL.Host == "gitlab.com" {- return gitlabBaseURL, nil- }-- // Beyond here, on-prem gitlab is being used, so we have to mostly leave things as-is.-- if gitlabURL.Scheme != "https" {- return "", fmt.Errorf("https was not used as URL scheme, but is required. Please use https")- }-- // The gitlab library wants trailing slashes.- if !strings.HasSuffix(gitlabURL.Path, "/") {- gitlabURL.Path = gitlabURL.Path + "/"- }-- return gitlabURL.String(), nil-}-// Enumerate reports all GitLab repositories to be scanned to the reporter. If// none are configured, it will find all repositories within all projects that// the configured user has access to, while respecting the configured ignore@@ -1157,11 +1122,19 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reportereturn nil}+ // Normalize before cloning so the git remote URL (used as the cache key+ // when attaching metadata to chunks) matches the key used when caching+ // project details below.+ normalizedRepoURL, err := giturl.NormalizeGitlabRepo(repoURL)+ if err != nil {+ ctx.Logger().Error(err, "failed to normalize GitLab Repo", "repo", repoURL)+ return err+ }+var path stringvar repo *gogit.Repository- var err errorif s.authMethod == "UNAUTHENTICATED" {- path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL, s.clonePath)+ path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, normalizedRepoURL, s.clonePath)} else {// If a username is not provided we need to use a default one in order to clone a private repo.// Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.@@ -1170,7 +1143,7 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporteuser = "placeholder"}- path, repo, err = git.CloneRepoUsingToken(ctx, s.token, repoURL, s.clonePath, user, s.useAuthInUrl)+ path, repo, err = git.CloneRepoUsingToken(ctx, s.token, normalizedRepoURL, s.clonePath, user, s.useAuthInUrl)}if err != nil {return err@@ -1186,28 +1159,30 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte// ensure project details are cached// this is required to populate metadata during chunking- s.ensureProjectInCache(ctx, repoURL)+ s.ensureProjectInCache(ctx, normalizedRepoURL)return s.git.ScanRepo(ctx, repo, path, s.scanOptions, reporter)}-// ensureProjectInCache checks if the project for the given repo URL is in the cache,-// and if not, queries the GitLab API to fetch the project and adds it to the cache.-func (s *Source) ensureProjectInCache(ctx context.Context, repoUrl string) {- // check if project is already in cache- if _, ok := s.repoToProjCache.get(repoUrl); ok {+// ensureProjectInCache ensures that the project for the given repository URL+// exists in the cache. If not, it fetches the project from the GitLab API+// and stores it in the cache.+func (s *Source) ensureProjectInCache(ctx context.Context, repoURL string) {+ // Check if the project is already cached.+ if _, ok := s.projectMetadataCache.Get(repoURL); ok {+ ctx.Logger().V(5).Info("cache hit: found project metadata in the cache", "cache_key", repoURL)return}- // query project- proj, err := s.getGitlabProject(ctx, repoUrl)+ // Fetch the project from GitLab.+ project, err := s.getGitlabProject(ctx, repoURL)if err != nil {- ctx.Logger().Error(err, "could not fetch project for repo", "repo", repoUrl)+ ctx.Logger().Error(err, "failed to fetch GitLab project", "repo", repoURL)return}- // add to cache- s.cacheGitlabProject(proj)+ // Cache the project metadata.+ s.cacheGitlabProjectMetadata(ctx, project)}func (s *Source) getGitlabProject(ctx context.Context, repoUrl string) (*gitlab.Project, error) {@@ -1230,16 +1205,100 @@ func (s *Source) getGitlabProject(ctx context.Context, repoUrl string) (*gitlab.return proj, nil}-func (s *Source) cacheGitlabProject(gitlabProj *gitlab.Project) {- proj := &project{- id: gitlabProj.ID,- name: gitlabProj.NameWithNamespace,+// cacheGitlabProjectMetadata caches GitLab project metadata keyed by the+// normalized GitLab repository URL.+func (s *Source) cacheGitlabProjectMetadata(ctx context.Context, glProject *gitlab.Project) {+ proj := &projectMetadata{+ id: int64(glProject.ID),+ name: glProject.NameWithNamespace,+ }++ if glProject.Owner != nil {+ if email := glProject.Owner.Email; email != "" {+ proj.owner = email+ } else {+ proj.owner = glProject.Owner.Username+ }+ }++ repoURL, err := giturl.NormalizeGitlabRepo(glProject.HTTPURLToRepo)+ if err != nil {+ ctx.Logger().Error(err, "failed to normalize GitLab Repo", "repo", glProject.HTTPURLToRepo)+ return+ }++ ctx.Logger().V(5).Info("cache set: added project metadata in the cache", "cache_key", repoURL)+ s.projectMetadataCache.Add(repoURL, proj)+}++func buildIgnorer(include, exclude []string, onCompile func(err error, pattern string)) func(repo string) bool {++ // compile and load globRepoFilter+ globRepoFilter := newGlobRepoFilter(include, exclude, onCompile)++ f := func(repo string) bool {+ if !globRepoFilter.includeRepo(repo) || globRepoFilter.ignoreRepo(repo) {+ return true+ }+ return false+ }++ return f+}++// normalizeRepos convert the repo urls from https://gitlab.com/org/repo -> https://gitlab.com/org/repo.git+func normalizeRepos(repos []string) ([]string, []error) {+ // Optimistically allocate space for all valid repositories.+ validRepos := make([]string, 0, len(repos))+ var errs []error+ for _, prj := range repos {+ repo, err := giturl.NormalizeGitlabRepo(prj)+ if err != nil {+ errs = append(errs, fmt.Errorf("unable to normalize gitlab repo url %q: %w", prj, err))+ continue+ }++ validRepos = append(validRepos, repo)+ }+ return validRepos, errs+}++// normalizeGitlabEndpoint ensures that if an endpoint is going to gitlab.com, we use https://gitlab.com/ as the endpoint.+// If we see the protocol is http, we error, because this shouldn't be used.+// Otherwise, it ensures we are using https as our protocol, if none was provided.+func normalizeGitlabEndpoint(gitlabEndpoint string) (string, error) {+ if gitlabEndpoint == "" {+ return gitlabCloudBaseURL, nil}- if gitlabProj.Owner != nil {- proj.owner = gitlabProj.Owner.Email- if proj.owner == "" {- proj.owner = gitlabProj.Owner.Username++ gitlabURL, err := url.Parse(gitlabEndpoint)+ if err != nil {+ return "", err+ }++ // We probably didn't receive a URL with a scheme, which messed up the parsing.+ if gitlabURL.Host == "" {+ gitlabURL, err = url.Parse("https://" + gitlabEndpoint)+ if err != nil {+ return "", err}}- s.repoToProjCache.set(gitlabProj.HTTPURLToRepo, proj)++ // If the host is gitlab.com, this is the cloud version, which has only one valid endpoint.+ if gitlabURL.Host == "gitlab.com" {+ return gitlabCloudBaseURL, nil+ }++ // Beyond here, on-prem gitlab is being used, so we have to mostly leave things as-is.++ if gitlabURL.Scheme != "https" {+ return "", fmt.Errorf("https was not used as URL scheme, but is required. Please use https")+ }++ // The gitlab library wants trailing slashes.+ if !strings.HasSuffix(gitlabURL.Path, "/") {+ gitlabURL.Path = gitlabURL.Path + "/"+ }++ return gitlabURL.String(), nil}diff --git a/pkg/sources/gitlab/gitlab_integration_test.go b/pkg/sources/gitlab/gitlab_integration_test.goindex 423acac29..c07c43172 100644--- a/pkg/sources/gitlab/gitlab_integration_test.go+++ b/pkg/sources/gitlab/gitlab_integration_test.go@@ -794,7 +794,7 @@ func TestSource_Enumerate_ProjectDetailsInChunkMetadata(t *testing.T) {chunksCh := make(chan *sources.Chunk, 1)chanReporter := sources.ChanReporter{Ch: chunksCh}// Clear cache to force querying project details- clear(s.repoToProjCache.cache)+ s.projectMetadataCache.Purge()go func() {defer close(chunksCh)for _, unit := range testReporter.Units {@@ -859,7 +859,7 @@ func TestSource_Chunks_SimplifiedGitlabEnumeration(t *testing.T) {require.NotEmpty(t, testReporter.Units, "enumeration returned no units")// Clear project cache to force project-detail lookups during chunking- clear(s.repoToProjCache.cache)+ s.projectMetadataCache.Purge()// Channel-based reporter to capture emitted chunkschunksCh := make(chan *sources.Chunk, 1)diff --git a/pkg/sources/gitlab/gitlab_test.go b/pkg/sources/gitlab/gitlab_test.goindex 5a0848fc4..8eb4642a3 100644--- a/pkg/sources/gitlab/gitlab_test.go+++ b/pkg/sources/gitlab/gitlab_test.go@@ -141,24 +141,24 @@ func Test_normalizeGitlabEndpoint(t *testing.T) {wantErr bool}{"the cloud url should return the cloud url": {- inputEndpoint: gitlabBaseURL,- outputEndpoint: gitlabBaseURL,+ inputEndpoint: gitlabCloudBaseURL,+ outputEndpoint: gitlabCloudBaseURL,},"empty string should return the cloud url": {inputEndpoint: "",- outputEndpoint: gitlabBaseURL,+ outputEndpoint: gitlabCloudBaseURL,},"no scheme cloud url should return the cloud url": {inputEndpoint: "gitlab.com",- outputEndpoint: gitlabBaseURL,+ outputEndpoint: gitlabCloudBaseURL,},"no scheme cloud url with trailing slash should return the cloud url": {inputEndpoint: "gitlab.com/",- outputEndpoint: gitlabBaseURL,+ outputEndpoint: gitlabCloudBaseURL,},"http scheme cloud url with organization should return the cloud url": {inputEndpoint: "http://gitlab.com/trufflesec",- outputEndpoint: gitlabBaseURL,+ outputEndpoint: gitlabCloudBaseURL,},// On-prem endpoint testing."on-prem url should be unchanged": {diff --git a/pkg/sources/gitlab/project_cache.go b/pkg/sources/gitlab/project_cache.godeleted file mode 100644index 0fe492c13..000000000--- a/pkg/sources/gitlab/project_cache.go+++ /dev/null@@ -1,29 +0,0 @@-package gitlab--import "sync"--type project struct {- id int64- name string- owner string-}--type repoToProjectCache struct {- sync.RWMutex-- cache map[string]*project-}--func (r *repoToProjectCache) get(repo string) (*project, bool) {- r.RLock()- defer r.RUnlock()- proj, ok := r.cache[repo]- return proj, ok-}--func (r *repoToProjectCache) set(repo string, proj *project) {- r.Lock()- defer r.Unlock()-- r.cache[repo] = proj-}