Skip to content

Find, verify, and analyze leaked credentials

99.7 KB
Raw Blame
package github

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"math/rand/v2"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-git/go-git/v5/plumbing"
	"github.com/gobwas/glob"
	"github.com/google/go-github/v67/github"
	"github.com/shurcooL/githubv4"
	"golang.org/x/sync/errgroup"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/anypb"

	"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)

const (
	SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB

	unauthGithubOrgRateLimt = 30
	defaultPagination       = 100
	membersAppPagination    = 500
)

type Source struct {
	name string

	sourceID          sources.SourceID
	jobID             sources.JobID
	verify            bool
	orgsCache         cache.Cache[string]
	memberCache       map[string]struct{}
	repos             []string
	filteredRepoCache *filteredRepoCache
	repoInfoCache     repoInfoCache
	totalRepoSize     int // total size of all repos in kb

	useCustomContentWriter bool
	git                    *git.Git

	scanOptMu   sync.Mutex // protects the scanOptions
	scanOptions *git.ScanOptions

	conn                   *sourcespb.GitHub
	jobPool                *errgroup.Group
	resumeInfoMutex        sync.Mutex
	resumeInfoSlice        []string
	connector              Connector
	appInstallationsMu     sync.Mutex
	appInstallations       []*github.Installation
	appInstallationsLoaded bool

	includePRComments     bool
	includeIssueComments  bool
	ignoreGists           bool
	includeGistComments   bool
	commentsTimeframeDays uint32
	// commentRetryWait sleeps between retries of transient comment API
	// failures and returns false if the context ended first. Nil uses
	// sleepWithContext; tests inject a wait that returns immediately.
	commentRetryWait func(context.Context, time.Duration) bool

	sources.Progress

	useAuthInUrl bool // pass credentials in the repository urls for cloning
}

// --------------------------------------------------------------------------------
// RepoUnit and GistUnit are implementations of SourceUnit used during
// enumeration. The different types aren't strictly necessary, but are a bit
// more explicit and allow type checking/safety.

var _ sources.SourceUnit = (*RepoUnit)(nil)
var _ sources.SourceUnit = (*GistUnit)(nil)

type RepoUnit struct {
	Name string `json:"name"`
	URL  string `json:"url"`
	// InstallationID is the GitHub App installation that owns this repo, set
	// during scan-all-installations enumeration. Carrying it on the unit lets
	// ChunkUnit clone with the correct installation token without re-deriving
	// the mapping, which otherwise re-lists every installation's repos on every
	// scanned unit (see INT-790). Zero when the source is not an installation
	// scan or the unit predates this field.
	InstallationID int64 `json:"installation_id,omitempty"`
}

func (r RepoUnit) SourceUnitID() (string, sources.SourceUnitKind) { return r.URL, "repo" }
func (r RepoUnit) Display() string                                { return r.Name }

type GistUnit struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

func (g GistUnit) SourceUnitID() (string, sources.SourceUnitKind) { return g.URL, "gist" }
func (g GistUnit) Display() string                                { return g.Name }

// unitEnvelope is just the JSON equivalent of apipb.SourceUnit, so we can
// unmarshal source unit payloads.
type unitEnvelope struct {
	ID       string                 `json:"id"`
	Kind     sources.SourceUnitKind `json:"kind,omitempty"`
	Name     string                 `json:"display,omitempty"`
	UnitData []byte                 `json:"unit_data,omitempty"`
}

func (u unitEnvelope) SourceUnitID() (string, sources.SourceUnitKind) { return u.ID, u.Kind }
func (u unitEnvelope) Display() string                                { return u.Name }

func unmarshalSourceUnit[unitType sources.SourceUnit](data []byte) (unitType, error) {
	u := new(unitType)

	if err := json.Unmarshal(data, u); err != nil {
		return *new(unitType), err
	}

	return *u, nil
}

func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
	env, err := unmarshalSourceUnit[unitEnvelope](data)
	if err != nil {
		return nil, err
	}

	switch env.Kind {
	case "": // data is a raw SourceUnit, no envelope
		// Look for a raw RepoUnit first
		ru, err := unmarshalSourceUnit[RepoUnit](data)
		if err != nil {
			return nil, err
		}

		if ru.URL != "" { // This is the test for a valid RepoUnit
			return ru, nil
		}

		if env.ID == "" { // This is the test for a valid generic GitHub Source Unit
			return nil, errors.New("not a github source unit")
		}

		return env, nil
	case "repo":
		if len(env.UnitData) > 0 {
			return unmarshalSourceUnit[RepoUnit](env.UnitData)
		}
		return RepoUnit{Name: env.Name, URL: env.ID}, nil
	case "gist":
		if len(env.UnitData) > 0 {
			return unmarshalSourceUnit[GistUnit](env.UnitData)
		}
		return GistUnit{Name: env.Name, URL: env.ID}, nil
	default:
		return nil, fmt.Errorf("unrecognized source unit kind %q", env.Kind)
	}
}

// --------------------------------------------------------------------------------

// WithCustomContentWriter sets the useCustomContentWriter flag on the source.
func (s *Source) WithCustomContentWriter() { s.useCustomContentWriter = true }

func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
	s.scanOptions = scanOptions
}

func (s *Source) setScanOptions(base, head string) {
	s.scanOptMu.Lock()
	defer s.scanOptMu.Unlock()
	s.scanOptions.BaseHash = base
	s.scanOptions.HeadHash = head
}

// Ensure the Source satisfies the interfaces at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumChunker = (*Source)(nil)

// Type returns the type of source.
// It is used for matching source types in configuration and job input.
func (s *Source) Type() sourcespb.SourceType {
	return SourceType
}

func (s *Source) SourceID() sources.SourceID {
	return s.sourceID
}

func (s *Source) JobID() sources.JobID {
	return s.jobID
}

// filteredRepoCache is a wrapper around cache.Cache that filters out repos
// based on include and exclude globs.
type filteredRepoCache struct {
	cache.Cache[string]
	include, exclude []glob.Glob
}

func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string], include, exclude []string) *filteredRepoCache {
	includeGlobs := make([]glob.Glob, 0, len(include))
	excludeGlobs := make([]glob.Glob, 0, len(exclude))
	for _, ig := range include {
		g, err := glob.Compile(ig)
		if err != nil {
			// the bad glob is dropped, so tell the user this filter is not used
			ctx.Logger().Error(err, "invalid include glob; ignoring it", "include_value", ig)
			continue
		}
		includeGlobs = append(includeGlobs, g)
	}
	for _, eg := range exclude {
		g, err := glob.Compile(eg)
		if err != nil {
			// the bad glob is dropped, so tell the user this filter is not used
			ctx.Logger().Error(err, "invalid exclude glob; ignoring it", "exclude_value", eg)
			continue
		}
		excludeGlobs = append(excludeGlobs, g)
	}
	return &filteredRepoCache{Cache: c, include: includeGlobs, exclude: excludeGlobs}
}

// Set overrides the cache.Cache Set method to filter out repos based on
// include and exclude globs.
func (c *filteredRepoCache) Set(key, val string) {
	if c.ignoreRepo(key) {
		return
	}
	if !c.includeRepo(key) {
		return
	}
	c.Cache.Set(key, val)
}

func (c *filteredRepoCache) ignoreRepo(s string) bool {
	for _, g := range c.exclude {
		if g.Match(s) {
			return true
		}
	}
	return false
}

func (c *filteredRepoCache) includeRepo(s string) bool {
	if len(c.include) == 0 {
		return true
	}

	for _, g := range c.include {
		if g.Match(s) {
			return true
		}
	}
	return false
}

// wantRepo returns true if the repository should be included based on include/exclude patterns
func (c *filteredRepoCache) wantRepo(s string) bool {
	return !c.ignoreRepo(s) && c.includeRepo(s)
}

// Init returns an initialized GitHub source.
func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
	err := git.CmdCheck()
	if err != nil {
		return err
	}

	s.name = name
	s.sourceID = sourceID
	s.jobID = jobID
	s.verify = verify
	s.jobPool = &errgroup.Group{}
	s.jobPool.SetLimit(concurrency)

	// Setup scan options if it wasn't provided.
	if s.scanOptions == nil {
		s.scanOptions = &git.ScanOptions{}
	}

	var conn sourcespb.GitHub
	err = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
	if err != nil {
		return fmt.Errorf("error unmarshalling connection: %w", err)
	}
	s.conn = &conn

	// configuration uses the inverse logic of the `useAuthInUrl` flag.
	s.useAuthInUrl = !s.conn.RemoveAuthInUrl
	s.resetAppInstallations()

	connector, err := newConnector(aCtx, s)
	if err != nil {
		return fmt.Errorf("could not create connector: %w", err)
	}
	s.connector = connector

	s.orgsCache = simple.NewCache[string]()
	for _, org := range s.conn.Organizations {
		s.orgsCache.Set(org, org)
	}
	s.memberCache = make(map[string]struct{})

	s.filteredRepoCache = s.newFilteredRepoCache(aCtx,
		simple.NewCache[string](),
		append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
		s.conn.GetIgnoreRepos(),
	)
	s.repos = s.conn.Repositories
	for _, repo := range s.repos {
		// safeRepoURL keeps the credential out of the lines we write, this
		// also catches it in any string field logged elsewhere
		log.RedactGlobally(repoCredential(repo))
		r, err := s.normalizeRepo(repo)
		if err != nil {
			// the repo comes from the user's config and may hold a token, so strip it,
			// also from the error, which repeats the URL
			safeErr := errors.New(strings.ReplaceAll(err.Error(), repo, safeRepoURL(repo)))
			aCtx.Logger().Error(safeErr, "invalid repository", "repo", safeRepoURL(repo))
			continue
		}
		s.filteredRepoCache.Set(repo, r)
	}
	s.repoInfoCache = newRepoInfoCache()

	s.includeIssueComments = s.conn.IncludeIssueComments
	s.includePRComments = s.conn.IncludePullRequestComments
	s.ignoreGists = s.conn.GetIgnoreGists()
	s.includeGistComments = s.conn.IncludeGistComments
	s.commentsTimeframeDays = s.conn.CommentsTimeframeDays

	// Head or base should only be used with incoming webhooks
	if (len(s.conn.Head) > 0 || len(s.conn.Base) > 0) && len(s.repos) != 1 {
		return fmt.Errorf("cannot specify head or base with multiple repositories")
	}

	if connector, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations {
		if _, err := s.getAppInstallations(aCtx, connector); err != nil {
			return err
		}
		if len(s.repos) > 0 {
			if err := s.mapExplicitReposToInstallations(aCtx, connector); err != nil {
				return err
			}
		}
	}

	cfg := &git.Config{
		SourceName:   s.name,
		JobID:        s.jobID,
		SourceID:     s.sourceID,
		SourceType:   s.Type(),
		Verify:       s.verify,
		SkipBinaries: conn.GetSkipBinaries(),
		SkipArchives: conn.GetSkipArchives(),
		Concurrency:  concurrency,
		SourceMetadataFunc: func(info git.SourceMetadataInfo) *source_metadatapb.MetaData {
			return &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Github{
					Github: &source_metadatapb.Github{
						Commit:              sanitizer.UTF8(info.Commit),
						File:                sanitizer.UTF8(info.File),
						Email:               sanitizer.UTF8(info.Email),
						Repository:          sanitizer.UTF8(info.Repository),
						Link:                giturl.GenerateLink(info.Repository, info.Commit, info.File, info.Line),
						Timestamp:           sanitizer.UTF8(info.Timestamp),
						Line:                info.Line,
						Visibility:          s.visibilityOf(aCtx, info.Repository),
						RepositoryLocalPath: sanitizer.UTF8(info.RepositoryLocalPath),
					},
				},
			}
		},
		UseCustomContentWriter: s.useCustomContentWriter,
		AuthInUrl:              s.useAuthInUrl,
	}
	s.git = git.NewGit(cfg)

	// log the config choices so support can see how the source was set up
	// only counts and flags, never repo URLs, tokens or keys
	aCtx.Logger().V(1).Info("github source initialized",
		"auth_type", fmt.Sprintf("%T", s.conn.GetCredential()),
		"endpoint", safeRepoURL(s.conn.GetEndpoint()),
		"repo_count", len(s.conn.GetRepositories()),
		"org_count", len(s.conn.GetOrganizations()),
		"include_repo_count", len(s.conn.GetIncludeRepos()),
		"ignore_repo_count", len(s.conn.GetIgnoreRepos()),
		"include_forks", s.conn.GetIncludeForks(),
		"exclude_archived", s.conn.GetExcludeArchived(),
		"scan_users", s.conn.GetScanUsers(),
		"scan_all_installations", s.conn.GetScanAllInstallations(),
		"ignore_gists", s.ignoreGists,
		"include_wikis", s.conn.GetIncludeWikis(),
		"include_issue_comments", s.includeIssueComments,
		"include_pr_comments", s.includePRComments,
		"include_gist_comments", s.includeGistComments,
		"comments_timeframe_days", s.commentsTimeframeDays,
	)
	return nil
}

// Validate is used by enterprise CLI to validate the GitHub config file.
func (s *Source) Validate(ctx context.Context) []error {
	/*
		Uses the rate limit API (docs: https://docs.github.com/en/rest/rate-limit) because:
		- Works with all auth types: user tokens, PATs, App credentials, and unauthenticated requests
		- Returns 401 for invalid credentials but works with no auth (as unauthenticated)
		- Doesn't consume API quota when called
	*/
	// When scanAllInstallations is set with no githubApp.installationId,
	// there's no default installation token to call RateLimit with (see
	// appConnector.HasDefaultInstallation). Validate the App credentials
	// directly against the app-level (JWT-authenticated) client instead.
	if connector, ok := s.connector.(*appConnector); ok && !connector.HasDefaultInstallation() {
		if _, _, err := connector.InstallationClient().Apps.Get(ctx, ""); err != nil {
			return []error{err}
		}
		return nil
	}

	if _, _, err := s.connector.APIClient().RateLimit.Get(ctx); err != nil {
		return []error{err}
	}

	return nil
}

func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metadatapb.Visibility {
	// It isn't possible to get the visibility of a wiki.
	// We must use the visibility of the corresponding repository.
	if _, ok := s.repoInfoCache.get(repoURL); !ok {
		if parentRepoURL, isWiki := repoCloneURLForWikiCloneURL(repoURL); isWiki {
			repoURL = parentRepoURL
		}
	}

	repoInfo, ok := s.repoInfoCache.get(repoURL)
	if !ok {
		// This should never happen.
		err := fmt.Errorf("no repoInfo for URL: %s", repoURL)
		ctx.Logger().Error(err, "failed to get repository visibility")
		return source_metadatapb.Visibility_unknown
	}

	return repoInfo.visibility
}

// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error {
	chunksReporter := sources.ChanReporter{Ch: chunksChan}
	// If targets are provided, we're only scanning the data in those targets.
	// Otherwise, we're scanning all data.
	// This allows us to only scan the commit where a vulnerability was found.
	if len(targets) > 0 {
		errs := s.scanTargets(ctx, targets, chunksReporter)
		return errors.Join(errs...)
	}

	// Reset consumption and rate limit metrics on each run.
	githubNumRateLimitEncountered.WithLabelValues(s.name).Set(0)
	githubSecondsSpentRateLimited.WithLabelValues(s.name).Set(0)
	githubReposScanned.WithLabelValues(s.name).Set(0)

	// We don't care about handling enumerated values as they happen during
	// the normal Chunks flow because we enumerate and scan in two steps.
	var enumerationErrs []error
	noopReporter := sources.VisitorReporter{
		VisitUnit: func(context.Context, sources.SourceUnit) error {
			return nil
		},
		VisitErr: func(ctx context.Context, err error) error {
			enumerationErrs = append(enumerationErrs, err)
			return ctx.Err()
		},
	}
	err := s.Enumerate(ctx, noopReporter)
	if err != nil {
		return errors.Join(fmt.Errorf("error enumerating: %w", err), errors.Join(enumerationErrs...))
	}

	return errors.Join(append(enumerationErrs, s.scan(ctx, chunksReporter))...)
}

// Enumerate enumerates the GitHub source based on authentication method and
// user configuration. It populates s.filteredRepoCache, s.repoInfoCache,
// s.memberCache, s.totalRepoSize, s.orgsCache, and s.repos. Additionally,
// repositories and gists are reported to the provided UnitReporter.
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
	seenUnits := make(map[sources.SourceUnit]struct{})
	// Wrapper reporter to deduplicate and filter found units.
	dedupeReporter := sources.VisitorReporter{
		VisitUnit: func(ctx context.Context, su sources.SourceUnit) error {
			// Only report units that passed the user configured filter.
			name := su.Display()
			if !s.filteredRepoCache.Exists(name) {
				return ctx.Err()
			}
			// Only report a unit once.
			if _, ok := seenUnits[su]; ok {
				return ctx.Err()
			}
			seenUnits[su] = struct{}{}
			return reporter.UnitOk(ctx, su)
		},
		VisitErr: reporter.UnitErr,
	}
	// Report any values that were already configured.
	// This compensates for differences in enumeration logic between `--org` and `--repo`.
	// See: https://github.com/trufflesecurity/trufflehog/pull/2379#discussion_r1487454788
	for _, name := range s.filteredRepoCache.Keys() {
		url, _ := s.filteredRepoCache.Get(name)
		url, err := s.ensureRepoInfoCache(ctx, url, &unitErrorReporter{reporter})
		if err != nil {
			if err := dedupeReporter.UnitErr(ctx, err); err != nil {
				return err
			}
			continue
		}

		var installationID int64 = 0
		if ac, ok := s.connector.(*appConnector); ok {
			ac.ensureRepoInstallation(url, name)
			installationID, _ = ac.installationIDForRepo(url)
		}

		ru := RepoUnit{Name: name, URL: url, InstallationID: installationID}

		if err := dedupeReporter.UnitOk(ctx, ru); err != nil {
			return err
		}
	}

	// I'm not wild about switching on the connector type here (as opposed to dispatching to the connector itself) but
	// this felt like a compromise that allowed me to isolate connection logic without rewriting the entire source.
	switch c := s.connector.(type) {
	case *appConnector:
		if err := s.enumerateWithApp(ctx, c, dedupeReporter); err != nil {
			return err
		}
	case *basicAuthConnector:
		if err := s.enumerateBasicAuth(ctx, dedupeReporter); err != nil {
			return err
		}
	case *tokenConnector:
		if err := s.enumerateWithToken(ctx, c.IsGithubEnterprise(), dedupeReporter); err != nil {
			return err
		}
	case *unauthenticatedConnector:
		s.enumerateUnauthenticated(ctx, dedupeReporter)
	}
	// If explicit repositories were provided, use them directly without filtering
	// Otherwise, rebuild s.repos from the filteredRepoCache
	if len(s.conn.Repositories) > 0 {
		// Explicit repositories bypass filtering - use them as-is
		s.repos = s.conn.Repositories
		ctx.Logger().V(1).Info("Using explicit repositories", "count", len(s.repos))
	} else {
		// No explicit repositories - rebuild from enumerated cache with filtering
		s.repos = make([]string, 0, s.filteredRepoCache.Count())

		// Double make sure that all enumerated repositories in the
		// filteredRepoCache have an entry in the repoInfoCache.
		for _, repo := range s.filteredRepoCache.Values() {
			// Extract the repository name from the URL for filtering
			repoName := extractRepoNameFromUrl(repo)

			// Final filter check - only include repositories that pass the filter
			if s.filteredRepoCache.wantRepo(repoName) {
				ctx := context.WithValue(ctx, "repo", safeRepoURL(repo))

				repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
				if err != nil {
					ctx.Logger().Error(err, "error caching repo info")
					_ = dedupeReporter.UnitErr(ctx, fmt.Errorf("error caching repo info: %w", err))
				}
				s.repos = append(s.repos, repo)
			}
		}
	}
	githubReposEnumerated.WithLabelValues(s.name).Set(float64(len(s.repos)))
	ctx.Logger().Info("Completed enumeration", "num_repos", len(s.repos), "num_orgs", s.orgsCache.Count(), "num_members", len(s.memberCache))
	// We must sort the repos so we can resume later if necessary.
	sort.Strings(s.repos)
	return nil
}

// ensureRepoInfoCache checks that s.repoInfoCache has an entry for the
// provided repository URL. If not, it fetches and stores the metadata for the
// repository. In some cases, the gist URL needs to be normalized, which is
// returned by this function.
func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string, reporter errorReporter) (string, error) {
	if _, ok := s.repoInfoCache.get(repo); ok {
		return repo, nil
	}
	ctx.Logger().V(2).Info("Caching repository info", "repo", safeRepoURL(repo))

	_, urlParts, err := getRepoURLParts(repo)
	if err != nil {
		return repo, fmt.Errorf("failed to parse repository URL: %w", err)
	}

	if !s.ignoreGists && isGistUrl(urlParts) {
		if connector, ok := s.connector.(*appConnector); ok && !connector.HasDefaultInstallation() {
			return repo, fmt.Errorf("cannot fetch gist %q: githubApp.installationId is required as a fallback installation to scan gists together with scanAllInstallations", repo)
		}
		// Cache gist info.
		for {
			gistID := extractGistID(urlParts)
			gist, _, err := s.connector.APIClient().Gists.Get(ctx, gistID)
			// Normalize the URL to the Gist's pull URL.
			// See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937
			repo = gist.GetGitPullURL()

			if s.handleRateLimit(ctx, err, reporter) {
				continue
			}

			if err != nil {
				return repo, fmt.Errorf("failed to fetch gist: %w", err)
			}

			s.cacheGistInfo(gist)
			break
		}
	} else {
		// Cache repository info.
		for {
			apiClient, err := s.connector.APIClientForRepo(repo)
			if err != nil {
				return repo, fmt.Errorf("failed to create API client for repository: %w", err)
			}

			ghRepo, _, err := apiClient.Repositories.Get(ctx, urlParts[1], urlParts[2])
			if s.handleRateLimit(ctx, err, reporter) {
				continue
			}
			if err != nil {
				return repo, fmt.Errorf("failed to fetch repository: %w", err)
			}
			s.cacheRepoInfo(ghRepo)
			// Handle repository redirects: the API returns the canonical CloneURL
			// which may differ from the original URL (e.g. org rename). Cache info
			// under the original URL too so scanRepo can look it up.
			if cloneURL := ghRepo.GetCloneURL(); cloneURL != repo {
				if info, ok := s.repoInfoCache.get(cloneURL); ok {
					s.repoInfoCache.put(repo, info)
				}
			}
			break
		}
	}
	return repo, nil
}

func (s *Source) enumerateBasicAuth(ctx context.Context, reporter sources.UnitReporter) error {
	for _, org := range s.orgsCache.Keys() {
		orgCtx := context.WithValue(ctx, "account", org)
		userType, err := s.getReposByOrgOrUser(ctx, org, true, reporter)
		if err != nil {
			orgCtx.Logger().Error(err, "error fetching repos for org or user")
			continue
		}

		// TODO: This modifies s.memberCache but it doesn't look like
		// we do anything with it.
		if userType == organization && s.conn.ScanUsers {
			if err := s.addMembersByOrg(ctx, org, reporter); err != nil {
				orgCtx.Logger().Error(err, "Unable to add members by org")
			}
		}
	}

	return nil
}

func (s *Source) enumerateUnauthenticated(ctx context.Context, reporter sources.UnitReporter) {
	if s.orgsCache.Count() > unauthGithubOrgRateLimt {
		ctx.Logger().Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
	}

	for _, org := range s.orgsCache.Keys() {
		orgCtx := context.WithValue(ctx, "account", org)
		userType, err := s.getReposByOrgOrUser(ctx, org, false, reporter)
		if err != nil {
			orgCtx.Logger().Error(err, "error fetching repos for org or user")
			continue
		}

		if userType == organization && s.conn.ScanUsers {
			orgCtx.Logger().Info("WARNING: Enumerating unauthenticated does not support scanning organization members (--include-members)")
		}
	}
}

func (s *Source) enumerateWithToken(ctx context.Context, isGithubEnterprise bool, reporter sources.UnitReporter) error {
	ctx.Logger().V(1).Info("Enumerating with token")

	var ghUser *github.User
	var err error
	for {
		ghUser, _, err = s.connector.APIClient().Users.Get(ctx, "")
		if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
			continue
		}
		if err != nil {
			return fmt.Errorf("error getting user: %w", err)
		}
		break
	}

	specificScope := len(s.repos) > 0 || s.orgsCache.Count() > 0
	if !specificScope {
		// Enumerate the user's orgs and repos if none were specified.
		if err := s.getReposByUser(ctx, ghUser.GetLogin(), true, reporter); err != nil {
			ctx.Logger().Error(err, "Unable to fetch repos for the current user", "user", ghUser.GetLogin())
		}
		if err := s.addUserGistsToCache(ctx, ghUser.GetLogin(), reporter); err != nil {
			ctx.Logger().Error(err, "Unable to fetch gists for the current user", "user", ghUser.GetLogin())
		}

		if isGithubEnterprise {
			s.addAllVisibleOrgs(ctx, reporter)
		} else {
			// Scan for orgs is default with a token.
			// GitHub App enumerates the repos that were assigned to it in GitHub App settings.
			s.addOrgsByUser(ctx, ghUser.GetLogin(), reporter)
		}
	}

	if len(s.orgsCache.Keys()) > 0 {
		for _, org := range s.orgsCache.Keys() {
			orgCtx := context.WithValue(ctx, "account", org)
			userType, err := s.getReposByOrgOrUser(ctx, org, true, reporter)
			if err != nil {
				orgCtx.Logger().Error(err, "Unable to fetch repos for org or user")
				continue
			}

			if userType == organization && s.conn.ScanUsers {
				if err := s.addMembersByOrg(ctx, org, reporter); err != nil {
					orgCtx.Logger().Error(err, "Unable to add members for org")
				}
			}
		}

		if s.conn.ScanUsers && len(s.memberCache) > 0 {
			s.addReposForMembers(ctx, reporter)
		}
	}

	return nil
}

func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector, reporter sources.UnitReporter) error {
	// If no repos were provided, enumerate them.
	if len(s.repos) == 0 {
		if s.conn.ScanAllInstallations {
			if err := s.enumerateAllInstallationRepos(ctx, connector, reporter); err != nil {
				return err
			}
		} else {
			if err := s.getReposByApp(ctx, reporter); err != nil {
				return err
			}
		}

		// Check if we need to find user repos.
		if s.conn.ScanUsers {
			err := s.addMembersByApp(ctx, connector, reporter)
			if err != nil {
				return err
			}
			ctx.Logger().V(1).Info("Scanning repos", "org_members", len(s.memberCache))
			if !connector.HasDefaultInstallation() {
				// Member repos/gists live outside any installation's own
				// listing, so there is no installation token to fetch them
				// with (see appConnector.HasDefaultInstallation). Skip with a
				// clear message instead of letting each per-member API call
				// fail against installation ID 0.
				ctx.Logger().Info("Skipping org member repos and gists: githubApp.installationId is required as a fallback installation to scan members together with scanAllInstallations", "org_members", len(s.memberCache))
				return nil
			}
			// Members' gists aren't listed, unlike in addReposForMembers: none
			// of GitHub's gist endpoints accept an installation access token,
			// and no app permission changes that, so the listing fails with a
			// 403 for every member. One line here stands in for those
			// failures; there's nothing for the customer to grant.
			if !s.ignoreGists {
				ctx.Logger().Info("Skipping org member gists: GitHub doesn't allow listing gists with a GitHub App installation token", "org_members", len(s.memberCache))
			}
			for member := range s.memberCache {
				// once the scan is cancelled every fetch fails, so stop here
				// instead of logging an error for each remaining member
				if common.IsDone(ctx) {
					ctx.Logger().V(4).Info("skipping remaining members", "reason", "scan cancelled")
					return nil
				}
				// add the member to the error line below so we know whose fetch failed
				memberCtx := context.WithValues(ctx, "member", member)
				// TODO: Add authenticated user list repo for app token. It does support as per docs but need to test it before we enable it here.
				// docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-the-authenticated-user
				if err := s.getReposByUser(ctx, member, false, reporter); err != nil && ctx.Err() == nil {
					memberCtx.Logger().Error(err, "error fetching repos by user")
				}
			}
		}
	}

	return nil
}

func (s *Source) resetAppInstallations() {
	s.appInstallationsMu.Lock()
	defer s.appInstallationsMu.Unlock()

	s.appInstallations = nil
	s.appInstallationsLoaded = false
}

func (s *Source) getAppInstallations(ctx context.Context, connector *appConnector) ([]*github.Installation, error) {
	s.appInstallationsMu.Lock()
	defer s.appInstallationsMu.Unlock()

	if s.appInstallationsLoaded {
		return s.appInstallations, nil
	}

	installs, err := s.listAppInstallations(ctx, connector, defaultPagination)
	if err != nil {
		return nil, err
	}
	s.appInstallations = installs
	s.appInstallationsLoaded = true
	return s.appInstallations, nil
}

func (s *Source) listAppInstallations(
	ctx context.Context,
	connector *appConnector,
	perPage int,
) ([]*github.Installation, error) {
	installationClient := connector.InstallationClient()
	opts := &github.ListOptions{PerPage: perPage}

	var allInstalls []*github.Installation
	for {
		installs, res, err := installationClient.Apps.ListInstallations(ctx, opts)
		if s.handleRateLimit(ctx, err) {
			continue
		}
		if err != nil {
			return nil, fmt.Errorf("could not list installations: %w", err)
		}
		allInstalls = append(allInstalls, installs...)
		if res == nil || res.NextPage == 0 {
			break
		}
		opts.Page = res.NextPage
	}

	return allInstalls, nil
}

func (s *Source) listAppInstallationRepoPage(
	ctx context.Context,
	client *github.Client,
	opts *github.ListOptions,
	handleRateLimit func(error) bool,
) ([]*github.Repository, *github.Response, error) {
	for {
		result, res, err := client.Apps.ListRepos(ctx, opts)
		if handleRateLimit(err) {
			continue
		}
		if result == nil {
			return nil, res, err
		}
		if result.Repositories == nil {
			return []*github.Repository{}, res, err
		}
		return result.Repositories, res, err
	}
}

// enumerateAllInstallationRepos discovers repos from every installation of the
// GitHub App. For each installation, it creates a per-installation API client
// and records the installation ID for each repo so Clone uses the correct token.
// Per-installation failures are reported through the unit reporter and don't
// abort enumeration of the remaining installations.
func (s *Source) enumerateAllInstallationRepos(ctx context.Context, connector *appConnector, reporter sources.UnitReporter) error {
	installs, err := s.getAppInstallations(ctx, connector)
	if err != nil {
		return err
	}

	for _, install := range installs {
		installID := install.GetID()
		account := install.GetAccount().GetLogin()
		installCtx := context.WithValues(ctx, "org", account, "installation_id", installID)
		installCtx.Logger().V(2).Info("Enumerating repos from GitHub App installation")

		client, err := connector.APIClientForInstallation(installCtx, installID)
		if err != nil {
			installCtx.Logger().Error(err, "could not create API client for installation")
			if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", installID, err)); err != nil {
				return err
			}
			continue
		}

		listRepos := func(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) {
			repos, resp, err := s.listAppInstallationRepoPage(ctx, client, opts.getListOptions(), func(err error) bool {
				return s.handleRateLimitWithUnitReporter(ctx, reporter, err)
			})
			for _, r := range repos {
				connector.setRepoInstallationForRepoName(r.GetCloneURL(), r.GetName(), installID)
			}
			return repos, resp, err
		}

		if err := s.processRepos(installCtx, account, reporter, listRepos, &appListOptions{
			ListOptions: github.ListOptions{PerPage: defaultPagination},
		}); err != nil {
			installCtx.Logger().Error(err, "error enumerating repos for installation")
			if err := reporter.UnitErr(installCtx, fmt.Errorf("error enumerating repos for installation %d: %w", installID, err)); err != nil {
				return err
			}
			continue
		}
	}

	return nil
}

func (s *Source) mapExplicitReposToInstallations(ctx context.Context, connector *appConnector) error {
	return s.mapReposToInstallations(ctx, connector, s.repos)
}

type repoMappingRequest struct {
	original   string
	normalized string
	lookupURLs []string
	lookupKeys []string
}

func (s *Source) mapReposToInstallations(ctx context.Context, connector *appConnector, repos []string) error {
	if len(repos) == 0 {
		return nil
	}

	wantedRepos := make(map[string]repoMappingRequest, len(repos))
	for _, repo := range repos {
		normalized, err := s.normalizeRepo(repo)
		if err != nil {
			return fmt.Errorf("could not normalize configured repo %q: %w", repo, err)
		}
		if connector.hasRepoInstallation(repo) || connector.hasRepoInstallation(normalized) {
			continue
		}
		request := repoMappingRequest{
			original:   repo,
			normalized: normalized,
			lookupURLs: repoURLsForInstallationLookup(normalized),
		}
		for _, lookupURL := range request.lookupURLs {
			key, err := repoHostOwnerNameKey(lookupURL)
			if err != nil {
				return fmt.Errorf("could not parse configured repo %q: %w", repo, err)
			}
			request.lookupKeys = append(request.lookupKeys, key)
		}
		// During installation repo listing, only allow exact matches. If no
		// exact repo exists, mapRemainingReposByMetadata tries wiki-parent
		// fallbacks afterward.
		wantedRepos[request.lookupKeys[0]] = request
	}
	if len(wantedRepos) == 0 {
		return nil
	}

	installs, err := s.getAppInstallations(ctx, connector)
	if err != nil {
		return err
	}

	var errs error
	clients := make(map[int64]*github.Client, len(installs))
	installedRepoKeys := make(map[int64]map[string]struct{}, len(installs))
	for _, install := range installs {
		installID := install.GetID()
		client, err := connector.APIClientForInstallation(ctx, installID)
		if err != nil {
			errs = errors.Join(errs, fmt.Errorf("could not create API client for installation %d: %w", installID, err))
			continue
		}
		clients[installID] = client

		opts := &github.ListOptions{PerPage: defaultPagination}
		for {
			repos, res, err := s.listAppInstallationRepoPage(ctx, client, opts, func(err error) bool {
				return s.handleRateLimit(ctx, err)
			})
			if err != nil {
				errs = errors.Join(errs, fmt.Errorf("could not list repos for installation %d: %w", installID, err))
				break
			}

			if repos != nil {
				if installedRepoKeys[installID] == nil {
					installedRepoKeys[installID] = make(map[string]struct{}, len(repos))
				}
				for _, repo := range repos {
					key, err := repoHostOwnerNameKey(repo.GetCloneURL())
					if err == nil {
						installedRepoKeys[installID][key] = struct{}{}
					}
				}
				s.recordInstallationRepoMatches(connector, installID, repos, wantedRepos)
				if len(wantedRepos) == 0 {
					if errs != nil {
						ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos")
					}
					return nil
				}
			}

			if res == nil || res.NextPage == 0 {
				break
			}
			opts.Page = res.NextPage
		}
	}

	if len(wantedRepos) > 0 {
		errs = errors.Join(errs, s.mapRemainingReposByMetadata(ctx, connector, installs, clients, installedRepoKeys, wantedRepos))
		if len(wantedRepos) == 0 {
			if errs != nil {
				ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos")
			}
			return nil
		}
	}

	if len(wantedRepos) > 0 {
		errs = errors.Join(errs, s.mapRemainingAccessibleRepos(ctx, connector, wantedRepos))
		if len(wantedRepos) == 0 {
			// Every requested repo was mapped, so returning errs would fail
			// repos that succeeded; log partial enumeration errors instead.
			if errs != nil {
				ctx.Logger().Error(errs, "some installations could not be fully enumerated while mapping repos")
			}
			return nil
		}
	}

	if len(wantedRepos) > 0 {
		unmatched := make([]string, 0, len(wantedRepos))
		for _, requested := range wantedRepos {
			unmatched = append(unmatched, requested.original)
		}
		sort.Strings(unmatched)
		errs = errors.Join(errs, fmt.Errorf("configured repos were not found in any GitHub App installation: %s", strings.Join(unmatched, ", ")))
	}

	return errs
}

func (s *Source) recordInstallationRepoMatches(connector *appConnector, installationID int64, repos []*github.Repository, wantedRepos map[string]repoMappingRequest) {
	for _, repo := range repos {
		cloneURL := repo.GetCloneURL()
		key, err := repoHostOwnerNameKey(cloneURL)
		if err != nil {
			continue
		}
		requested, ok := wantedRepos[key]
		if !ok {
			continue
		}
		deleteRepoMappingRequest(wantedRepos, requested)
		s.recordRepoInstallation(connector, installationID, repo, requested)
	}
}

func deleteRepoMappingRequest(wantedRepos map[string]repoMappingRequest, requested repoMappingRequest) {
	if len(requested.lookupKeys) > 0 {
		delete(wantedRepos, requested.lookupKeys[0])
	}
}

func (s *Source) mapRemainingReposByMetadata(
	ctx context.Context,
	connector *appConnector,
	installs []*github.Installation,
	clients map[int64]*github.Client,
	installedRepoKeys map[int64]map[string]struct{},
	wantedRepos map[string]repoMappingRequest,
) error {
	var errs error
	for _, install := range installs {
		if len(wantedRepos) == 0 {
			return errs
		}

		installID := install.GetID()
		client, ok := clients[installID]
		if !ok {
			continue
		}

		for _, requested := range wantedRepos {
			for i, lookupURL := range requested.lookupURLs {
				_, parts, err := getRepoURLParts(lookupURL)
				if err != nil {
					errs = errors.Join(errs, fmt.Errorf("could not parse configured repo %q: %w", requested.original, err))
					break
				}

				var repo *github.Repository
				for {
					repo, _, err = client.Repositories.Get(ctx, parts[1], parts[2])
					if s.handleRateLimit(ctx, err) {
						continue
					}
					break
				}
				if err != nil {
					if isGitHub404Error(err) {
						continue
					}
					errs = errors.Join(errs, fmt.Errorf("could not fetch repo %q for installation %d: %w", requested.original, installID, err))
					break
				}
				canonicalKey, err := repoHostOwnerNameKey(repo.GetCloneURL())
				if err != nil {
					errs = errors.Join(errs, fmt.Errorf("could not parse fetched repo %q for installation %d: %w", repo.GetCloneURL(), installID, err))
					continue
				}
				if _, ok := installedRepoKeys[installID][canonicalKey]; !ok {
					continue
				}
				if !sameRepoHost(requested.normalized, repo.GetCloneURL()) {
					if i == 0 {
						break
					}
					continue
				}

				deleteRepoMappingRequest(wantedRepos, requested)
				s.recordRepoInstallation(connector, installID, repo, requested)
				if len(wantedRepos) == 0 {
					return errs
				}
				break
			}
		}
	}
	return errs
}

// mapRemainingAccessibleRepos handles repos that no installation owns, such
// as org members' personal repos enumerated via scan_users. Installations
// only ever contain repos owned by the installing account, but the default
// installation token can still read repos that are visible to it (public
// repos in particular), so a repo that the API confirms accessible is mapped
// to the default installation instead of failing the scan.
func (s *Source) mapRemainingAccessibleRepos(
	ctx context.Context,
	connector *appConnector,
	wantedRepos map[string]repoMappingRequest,
) error {
	if !connector.HasDefaultInstallation() {
		return fmt.Errorf("cannot resolve %d configured repo(s) outside installation listings: githubApp.installationId is required as a fallback installation together with scanAllInstallations", len(wantedRepos))
	}

	var errs error
	client := connector.APIClient()
	for _, requested := range wantedRepos {
		for i, lookupURL := range requested.lookupURLs {
			_, parts, err := getRepoURLParts(lookupURL)
			if err != nil {
				errs = errors.Join(errs,
					fmt.Errorf("could not parse configured repo %q: %w", requested.original, err))
				break
			}

			var repo *github.Repository
			for {
				repo, _, err = client.Repositories.Get(ctx, parts[1], parts[2])
				if s.handleRateLimit(ctx, err) {
					continue
				}
				break
			}
			if err != nil {
				if isGitHub404Error(err) {
					continue
				}
				errs = errors.Join(errs,
					fmt.Errorf("could not fetch repo %q with default installation: %w", requested.original, err))
				break
			}
			if !sameRepoHost(requested.normalized, repo.GetCloneURL()) {
				if i == 0 {
					break
				}
				continue
			}

			// requested.original is the repo as the user wrote it, so strip any token
			ctx.Logger().Info("repo is not owned by any app installation; scanning with the default installation token",
				"repo", safeRepoURL(requested.original))
			deleteRepoMappingRequest(wantedRepos, requested)
			s.recordRepoInstallation(connector, connector.installationID, repo, requested)
			break
		}
	}
	return errs
}

func (s *Source) recordRepoInstallation(connector *appConnector, installationID int64, repo *github.Repository, requested repoMappingRequest) {
	cloneURL := repo.GetCloneURL()
	connector.setRepoInstallationForRepoName(cloneURL, repo.GetName(), installationID)
	if requested.normalized != cloneURL && sameRepoHost(requested.normalized, cloneURL) {
		connector.setRepoInstallation(requested.normalized, installationID)
	}
	if requested.original != requested.normalized && requested.original != cloneURL && sameRepoHost(requested.original, cloneURL) {
		connector.setRepoInstallation(requested.original, installationID)
	}

	s.cacheRepoInfo(repo)
	if info, ok := s.repoInfoCache.get(cloneURL); ok {
		if requested.normalized != cloneURL && sameRepoHost(requested.normalized, cloneURL) {
			s.repoInfoCache.put(requested.normalized, info)
		}
		if requested.original != requested.normalized && requested.original != cloneURL && sameRepoHost(requested.original, cloneURL) {
			s.repoInfoCache.put(requested.original, info)
		}
	}
}

func repoHostOwnerNameKey(repoURL string) (string, error) {
	_, parts, err := getRepoURLParts(repoURL)
	if err != nil {
		return "", err
	}
	if len(parts) != 3 {
		return "", fmt.Errorf("expected repository URL, got %q", repoURL)
	}
	return strings.ToLower(parts[0] + "/" + parts[1] + "/" + parts[2]), nil
}

func sameRepoHost(repoURL, otherRepoURL string) bool {
	_, parts, err := getRepoURLParts(repoURL)
	if err != nil {
		return false
	}
	_, otherParts, err := getRepoURLParts(otherRepoURL)
	if err != nil {
		return false
	}
	return strings.EqualFold(parts[0], otherParts[0])
}

func (s *Source) repoURLFromTargetMetadata(meta *source_metadatapb.Github) (string, error) {
	linkRepo, err := s.repoURLFromTargetLink(meta.GetLink())
	if err != nil {
		return "", err
	}

	if repo := meta.GetRepository(); repo != "" {
		// Target metadata may only include a bare repo name. In that case the
		// owner and host from the link are the authoritative repository identity.
		if !strings.Contains(repo, "/") && !strings.Contains(repo, ":") {
			return linkRepo, nil
		}
		normalized, err := s.normalizeRepo(repo)
		if err != nil {
			return "", fmt.Errorf("could not normalize target repository %q: %w", repo, err)
		}
		linkKey, err := repoHostOwnerNameKey(linkRepo)
		if err != nil {
			return "", err
		}
		repoKey, err := repoHostOwnerNameKey(normalized)
		if err != nil {
			return "", fmt.Errorf("could not parse target repository %q: %w", repo, err)
		}
		if repoKey != linkKey && isWikiLink(meta.GetLink()) {
			if parentRepoURL, isWiki := repoCloneURLForWikiCloneURL(normalized); isWiki {
				repoKey, err = repoHostOwnerNameKey(parentRepoURL)
				if err != nil {
					return "", fmt.Errorf("could not parse target repository %q: %w", repo, err)
				}
			}
		}
		if repoKey != linkKey {
			return "", fmt.Errorf("target repository %q does not match link repository %q", normalized, linkRepo)
		}
	}

	return linkRepo, nil
}

func (s *Source) repoURLFromTargetLink(link string) (string, error) {
	u, err := url.Parse(link)
	if err != nil {
		return "", fmt.Errorf("unable to parse GitHub URL: %w", err)
	}

	segments := strings.Split(strings.Trim(u.Path, "/"), "/")
	if len(segments) < 2 {
		return "", fmt.Errorf("invalid GitHub URL")
	}

	repoURL := (&url.URL{
		Scheme: u.Scheme,
		Host:   u.Host,
	}).JoinPath(segments[0], segments[1]+".git").String()
	return s.normalizeRepo(repoURL)
}

func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error {
	var scannedCount uint64 = 1

	ctx.Logger().V(1).Info("Found repos to scan", "count", len(s.repos))

	// If there is resume information available, limit this scan to only the repos that still need scanning.
	reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
	// repos finished by an earlier, interrupted run are left out, so say how many
	if skipped := len(s.repos) - len(reposToScan); skipped > 0 {
		ctx.Logger().V(1).Info("resuming scan; skipping repos already scanned", "skipped", skipped, "remaining", len(reposToScan))
	}
	s.repos = reposToScan

	for i, repoURL := range s.repos {
		s.jobPool.Go(func() error {
			// repos the user listed in the config are used as-is and may hold a
			// token, so only the cleaned URL goes into logs
			if common.IsDone(ctx) {
				ctx.Logger().V(4).Info("skipping repo", "repo", safeRepoURL(repoURL), "reason", "scan cancelled")
				return nil
			}
			ctx := context.WithValue(ctx, "repo", safeRepoURL(repoURL))

			// TODO: set progress complete is being called concurrently with i
			s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
			// Ensure the repo is removed from the resume info after being scanned.
			defer func(s *Source, repoURL string) {
				s.resumeInfoMutex.Lock()
				defer s.resumeInfoMutex.Unlock()
				s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)
			}(s, repoURL)

			if err := s.scanRepo(ctx, repoURL, reporter); err != nil {
				ctx.Logger().Error(err, "error scanning repo")
				return nil
			}

			atomic.AddUint64(&scannedCount, 1)
			return nil
		})
	}

	_ = s.jobPool.Wait()
	s.SetProgressComplete(len(s.repos), len(s.repos), "Completed GitHub scan", "")

	return nil
}

// scanRepo attempts to scan the provided URL and any associated wiki and
// comments if configured. An error is returned if we could not find necessary
// repository metadata or clone the repo, otherwise all errors are reported to
// the ChunkReporter.
func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.ChunkReporter) error {
	if !strings.HasSuffix(repoURL, ".git") {
		return fmt.Errorf("repo does not end in .git")
	}
	// Scan the repository
	repoInfo, ok := s.repoInfoCache.get(repoURL)
	if !ok {
		// This should never happen.
		return fmt.Errorf("no repoInfo for URL: %s", repoURL)
	}
	start := time.Now()
	gitDuration, err := s.cloneAndScanRepo(ctx, repoURL, repoInfo, reporter)
	if err != nil {
		return err
	}

	// Scan the wiki, if enabled, and the repo has one.
	// The checks are split up only so each skip can say why.
	switch {
	case !s.conn.IncludeWikis:
		// wikis are off for the whole source, nothing to log per repo
	case !repoInfo.hasWiki:
		ctx.Logger().V(4).Info("skipping wiki", "reason", "repo has no wiki")
	case !s.wikiIsReachable(ctx, repoURL):
		ctx.Logger().V(4).Info("skipping wiki", "reason", "wiki is not reachable")
	default:
		wikiURL, ok := wikiCloneURLForRepoInfo(repoURL, repoInfo)
		if !ok {
			ctx.Logger().V(4).Info("skipping wiki", "reason", "could not build wiki clone URL")
			break
		}
		wikiCtx := context.WithValue(ctx, "repo", safeRepoURL(wikiURL))

		_, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)
		if err != nil {
			// Ignore "Repository not found" errors.
			// It's common for GitHub's API to say a repo has a wiki when it doesn't.
			if strings.Contains(err.Error(), "not found") {
				wikiCtx.Logger().V(4).Info("skipping wiki", "reason", "GitHub says the wiki was not found")
			} else {
				if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil {
					return err
				}
			}

			// Don't return, it still might be possible to scan comments.
		}
	}

	// Scan comments, if enabled.
	var commentsDuration time.Duration
	if s.includeGistComments || s.includeIssueComments || s.includePRComments {
		commentsStart := time.Now()
		err := s.scanComments(ctx, repoURL, repoInfo, reporter)
		commentsDuration = time.Since(commentsStart)
		if err != nil {
			if err := reportCommentErrors(ctx, reporter, err); err != nil {
				return err
			}
		}
	}

	// duration_seconds covers everything above: clone, git history, wiki, and
	// comments. Git history and comments are broken out because on repos with
	// many comments, the comment walks can take far longer than the git scan.
	ctx.Logger().V(2).Info("finished scanning repo",
		"duration_seconds", time.Since(start).Seconds(),
		"git_scan_seconds", gitDuration.Seconds(),
		"comments_seconds", commentsDuration.Seconds())
	githubReposScanned.WithLabelValues(s.name).Inc()
	return nil
}

func (s *Source) cloneAndScanRepo(ctx context.Context, repoURL string, repoInfo repoInfo, reporter sources.ChunkReporter) (time.Duration, error) {
	var duration time.Duration

	ctx.Logger().V(2).Info("attempting to clone repo", "repo_name", repoInfo.name)
	path, repo, err := s.cloneRepo(ctx, repoURL)
	if err != nil {
		return duration, err
	}
	// remove the path only if it was created as a temporary path, or if it is a clone path and --no-cleanup is not set.
	// if legacy JSON is enabled, don't remove the directory because we need it for outputting legacy JSON.
	if !s.conn.GetPrintLegacyJson() {
		if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.conn.NoCleanup && s.conn.GetClonePath() != "") {
			defer func() { _ = os.RemoveAll(path) }()
		}
	}

	// TODO: Can this be set once or does it need to be set on every iteration? Is |s.scanOptions| set every clone?
	s.setScanOptions(s.conn.Base, s.conn.Head)

	start := time.Now()
	if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, reporter); err != nil {
		return duration, fmt.Errorf("error scanning repo %s: %w", repoURL, err)
	}
	duration = time.Since(start)
	return duration, nil
}

var (
	rateLimitMu         sync.RWMutex
	rateLimitResumeTime time.Time
)

// errorReporter is an interface that captures just the error reporting functionality
type errorReporter interface {
	Err(ctx context.Context, err error) error
}

// wrapper to adapt UnitReporter to errorReporter
type unitErrorReporter struct {
	reporter sources.UnitReporter
}

func (u unitErrorReporter) Err(ctx context.Context, err error) error {
	return u.reporter.UnitErr(ctx, err)
}

// wrapper to adapt ChunkReporter to errorReporter
type chunkErrorReporter struct {
	reporter sources.ChunkReporter
}

func (c chunkErrorReporter) Err(ctx context.Context, err error) error {
	return c.reporter.ChunkErr(ctx, err)
}

// handleRateLimit handles GitHub API rate limiting with an optional error reporter.
// Returns true if a rate limit was handled.
//
// Unauthenticated users have a rate limit of 60 requests per hour.
// Authenticated users have a rate limit of 5,000 requests per hour,
// however, certain actions are subject to a stricter "secondary" limit.
// https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api
func (s *Source) handleRateLimit(ctx context.Context, errIn error, reporters ...errorReporter) bool {
	if errIn == nil {
		return false
	}

	rateLimitMu.RLock()
	resumeTime := rateLimitResumeTime
	rateLimitMu.RUnlock()

	var retryAfter time.Duration
	if resumeTime.IsZero() || time.Now().After(resumeTime) {
		rateLimitMu.Lock()
		var (
			now = time.Now()

			// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
			limitType  string
			rateLimit  *github.RateLimitError
			abuseLimit *github.AbuseRateLimitError
		)
		if errors.As(errIn, &rateLimit) {
			limitType = "primary"
			rate := rateLimit.Rate
			if rate.Remaining == 0 { // TODO: Will we ever receive a |RateLimitError| when remaining > 0?
				retryAfter = rate.Reset.Sub(now)
			}
		} else if errors.As(errIn, &abuseLimit) {
			limitType = "secondary"
			retryAfter = abuseLimit.GetRetryAfter()
		} else {
			rateLimitMu.Unlock()
			return false
		}

		jitter := time.Duration(rand.IntN(10)+1) * time.Second
		if retryAfter > 0 {
			retryAfter = retryAfter + jitter
			rateLimitResumeTime = now.Add(retryAfter)
			ctx.Logger().Info("exceeded rate limit", "limit_type", limitType, "retry_after", retryAfter.String(), "resume_time", rateLimitResumeTime.Format(time.RFC3339))
			// Only report the error if a reporter was provided
			for _, reporter := range reporters {
				_ = reporter.Err(ctx, fmt.Errorf("exceeded %s rate limit", limitType))
			}
		} else {
			retryAfter = (5 * time.Minute) + jitter
			rateLimitResumeTime = now.Add(retryAfter)
			// TODO: Use exponential backoff instead of static retry time.
			ctx.Logger().Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", rateLimitResumeTime.Format(time.RFC3339))
		}

		rateLimitMu.Unlock()
	} else {
		retryAfter = time.Until(resumeTime)
	}

	githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
	slept, canceled := sleepWithContext(ctx, retryAfter)
	githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(slept.Seconds())
	return !canceled
}

func sleepWithContext(ctx context.Context, duration time.Duration) (time.Duration, bool) {
	start := time.Now()
	if duration <= 0 {
		return time.Since(start), false
	}

	timer := time.NewTimer(duration)
	defer timer.Stop()
	select {
	case <-timer.C:
		return time.Since(start), false
	case <-ctx.Done():
		return time.Since(start), true
	}
}

// handleRateLimitWithUnitReporter is a wrapper around handleRateLimit that includes unit reporting
func (s *Source) handleRateLimitWithUnitReporter(ctx context.Context, reporter sources.UnitReporter, errIn error) bool {
	return s.handleRateLimit(ctx, errIn, &unitErrorReporter{reporter: reporter})
}

// handleRateLimitWithChunkReporter is a wrapper around handleRateLimit that includes chunk reporting
func (s *Source) handleRateLimitWithChunkReporter(ctx context.Context, reporter sources.ChunkReporter, errIn error) bool {
	return s.handleRateLimit(ctx, errIn, &chunkErrorReporter{reporter: reporter})
}

func (s *Source) addReposForMembers(ctx context.Context, reporter sources.UnitReporter) {
	ctx.Logger().V(1).Info("Fetching repos from members", "members", len(s.memberCache), "org_count", s.orgsCache.Count())
	for member := range s.memberCache {
		// once the scan is cancelled every fetch fails, so stop here
		// instead of logging an error for each remaining member
		if common.IsDone(ctx) {
			ctx.Logger().V(4).Info("skipping remaining members", "reason", "scan cancelled")
			return
		}
		// a fetch that was running when the scan got cancelled fails too,
		// that is not an error worth logging
		if err := s.addUserGistsToCache(ctx, member, reporter); err != nil && ctx.Err() == nil {
			ctx.Logger().Error(err, "Unable to fetch gists by user", "user", member)
		}
		if err := s.getReposByUser(ctx, member, false, reporter); err != nil && ctx.Err() == nil {
			ctx.Logger().Error(err, "Unable to fetch repos by user", "user", member)
		}
	}
}

// addUserGistsToCache collects all the gist urls for a given user,
// and adds them to the filteredRepoCache.
func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporter sources.UnitReporter) error {
	if s.ignoreGists {
		return nil
	}

	gistOpts := &github.GistListOptions{}
	// every log line below is about this user
	ctx = context.WithValues(ctx, "user", user)

	for {
		gists, res, err := s.connector.APIClient().Gists.List(ctx, user, gistOpts)
		if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
			continue
		}
		if err != nil {
			return fmt.Errorf("could not list gists for user %s: %w", user, err)
		}

		for _, gist := range gists {
			s.filteredRepoCache.Set(gist.GetID(), gist.GetGitPullURL())
			s.cacheGistInfo(gist)
			if err := reporter.UnitOk(ctx, GistUnit{Name: gist.GetID(), URL: gist.GetGitPullURL()}); err != nil {
				return err
			}
		}

		if res == nil || res.NextPage == 0 {
			break
		}
		// one line per page, with how many API calls we have left before the rate limit
		ctx.Logger().V(4).Info("Listed gists", "page", gistOpts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)
		gistOpts.Page = res.NextPage
	}
	return nil
}

func (s *Source) addMembersByApp(ctx context.Context, connector *appConnector, reporter sources.UnitReporter) error {
	// TODO: Check rate limit for this call.
	installs, err := s.getAppInstallations(ctx, connector)
	if err != nil {
		return fmt.Errorf("could not enumerate installed orgs: %w", err)
	}

	for _, install := range installs {
		if install.GetAccount().GetType() != "Organization" {
			continue
		}
		org := install.GetAccount().GetLogin()
		installCtx := context.WithValues(ctx, "org", org, "installation_id", install.GetID())

		// Create a per-installation API client so that the token is scoped to
		// this org. GitHub App installation tokens only bypass IP allowlists for
		// their own org; using a cross-org token causes 403s.
		client, err := connector.APIClientForInstallation(installCtx, install.GetID())
		if err != nil {
			installCtx.Logger().Error(err, "could not create API client for installation")
			if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", install.GetID(), err)); err != nil {
				return err
			}
			continue
		}

		if err := s.addMembersByOrgWithClient(installCtx, client, org, reporter); err != nil {
			installCtx.Logger().Error(err, "Unable to add members for org")
			if err := reporter.UnitErr(installCtx, fmt.Errorf("could not add members for org %q (installation %d): %w", org, install.GetID(), err)); err != nil {
				return err
			}
			continue
		}
	}

	return nil
}

func (s *Source) addAllVisibleOrgs(ctx context.Context, reporter sources.UnitReporter) {
	ctx.Logger().V(2).Info("enumerating all visible organizations on GHE")
	// Enumeration on this endpoint does not use pages it uses a since ID.
	// The endpoint will return organizations with an ID greater than the given since ID.
	// Empty org response is our cue to break the enumeration loop.
	orgOpts := &github.OrganizationsListOptions{
		Since: 0,
		ListOptions: github.ListOptions{
			PerPage: defaultPagination,
		},
	}
	for {
		orgs, _, err := s.connector.APIClient().Organizations.ListAll(ctx, orgOpts)
		if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
			continue
		}
		if err != nil {
			ctx.Logger().Error(err, "could not list all organizations")
			return
		}

		if len(orgs) == 0 {
			break
		}

		lastOrgID := *orgs[len(orgs)-1].ID
		ctx.Logger().V(4).Info("listed organization IDs", "from_id", orgOpts.Since, "to_id", lastOrgID)
		orgOpts.Since = lastOrgID

		for _, org := range orgs {
			var name string
			switch {
			case org.Name != nil:
				name = *org.Name
			case org.Login != nil:
				name = *org.Login
			default:
				continue
			}
			s.orgsCache.Set(name, name)
			ctx.Logger().V(2).Info("adding organization for repository enumeration", "id", org.ID, "name", name)
		}
	}
}

func (s *Source) addOrgsByUser(ctx context.Context, user string, reporter sources.UnitReporter) {
	orgOpts := &github.ListOptions{
		PerPage: defaultPagination,
	}
	// every log line below is about this user
	ctx = context.WithValues(ctx, "user", user)
	for {
		orgs, resp, err := s.connector.APIClient().Organizations.List(ctx, "", orgOpts)
		if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
			continue
		}
		if err != nil {
			ctx.Logger().Error(err, "Could not list organizations")
			return
		}

		// one line per page, with how many API calls we have left before the rate limit
		ctx.Logger().V(4).Info("Listed orgs", "page", orgOpts.Page, "last_page", resp.LastPage, "rate_remaining", resp.Rate.Remaining)
		for _, org := range orgs {
			if org.Login == nil {
				continue
			}
			s.orgsCache.Set(*org.Login, *org.Login)
		}
		if resp.NextPage == 0 {
			break
		}
		orgOpts.Page = resp.NextPage
	}
}

func (s *Source) addMembersByOrg(ctx context.Context, org string, reporter sources.UnitReporter) error {
	return s.addMembersByOrgWithClient(ctx, s.connector.APIClient(), org, reporter)
}

func (s *Source) addMembersByOrgWithClient(ctx context.Context, client *github.Client, org string, reporter sources.UnitReporter) error {
	opts := &github.ListMembersOptions{
		PublicOnly: false,
		ListOptions: github.ListOptions{
			PerPage: membersAppPagination,
		},
	}

	// every log line below is about this org
	ctx = context.WithValues(ctx, "org", org)
	for {
		members, res, err := client.Organizations.ListMembers(ctx, org, opts)
		if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
			continue
		}
		if err != nil {
			return fmt.Errorf("could not list organization (%q) members: account may not have access to list organization members: %w", org, err)
		}
		if len(members) == 0 {
			return fmt.Errorf("organization (%q) had 0 members: account may not have access to list organization members", org)
		}

		// one line per page, with how many API calls we have left before the rate limit
		ctx.Logger().V(4).Info("Listed members", "page", opts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)
		for _, m := range members {
			usr := m.Login
			if usr == nil || *usr == "" {
				continue
			}
			if _, ok := s.memberCache[*usr]; !ok {
				s.memberCache[*usr] = struct{}{}
			}
		}
		if res.NextPage == 0 {
			break
		}
		opts.Page = res.NextPage
	}

	return nil
}

// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
	s.resumeInfoMutex.Lock()
	defer s.resumeInfoMutex.Unlock()

	// Add the repoURL to the resume info slice.
	s.resumeInfoSlice = append(s.resumeInfoSlice, repoURL)
	sort.Strings(s.resumeInfoSlice)

	// Make the resume info string from the slice.
	encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)

	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
}

// reportCommentErrors reports a failed comment scan as chunk errors, one per
// failed phase. processIssueandPRsWithCommentsREST joins one error per phase
// that failed, and each phase is a separate listing that failed for its own
// reason, often a different missing permission. Reporting them separately
// gives each its own entry and its own "error scanning comments" context,
// instead of one entry whose later lines have neither.
func reportCommentErrors(ctx context.Context, reporter sources.ChunkReporter, err error) error {
	phaseErrs := []error{err}
	if joined, ok := err.(interface{ Unwrap() []error }); ok {
		phaseErrs = joined.Unwrap()
	}
	for _, phaseErr := range phaseErrs {
		if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning comments: %w", phaseErr)); err != nil {
			return err
		}
	}
	return nil
}

func (s *Source) scanComments(ctx context.Context, repoPath string, repoInfo repoInfo, reporter sources.ChunkReporter) error {
	urlString, urlParts, err := getRepoURLParts(repoPath)
	if err != nil {
		return err
	}

	var cutoffTime *time.Time
	if s.commentsTimeframeDays > 0 {
		daysToFilter := int(s.commentsTimeframeDays)
		t := time.Now().AddDate(0, 0, -daysToFilter)
		cutoffTime = &t
	}

	apiClient, err := s.connector.APIClientForRepo(repoPath)
	if err != nil {
		return err
	}

	if s.includeGistComments && isGistUrl(urlParts) && !s.ignoreGists {
		return s.processGistComments(ctx, apiClient, urlString, urlParts, repoInfo, reporter, cutoffTime)
	} else if s.includeIssueComments || s.includePRComments {
		// if we need to use graphql api for repo issues, prs and comments
		if feature.UseGithubGraphQLAPI.Load() {
			graphqlClient, err := s.connector.GraphQLClientForRepo(ctx, repoPath)
			if err != nil {
				return err
			}
			return s.processRepoIssueandPRsWithCommentsGraphql(ctx, graphqlClient, repoInfo, reporter, cutoffTime)
		}

		return s.processIssueandPRsWithCommentsREST(ctx, apiClient, repoInfo, s.commentGrantsForRepo(ctx, repoPath), reporter, cutoffTime)
	}

	return nil
}

// commentGrantsForRepo returns what the GitHub App installation behind
// repoURL grants, or nil (unknown) when the source doesn't authenticate as an
// App. A failure to read the grants also gives nil: the listings then run
// without the precheck, as they would for a token, and any 403 is still
// classified from its response.
func (s *Source) commentGrantsForRepo(ctx context.Context, repoURL string) *installationGrants {
	app, ok := s.connector.(*appConnector)
	if !ok {
		return nil
	}
	grants, err := app.grantsForRepo(ctx, repoURL)
	if err != nil {
		ctx.Logger().V(2).Info("could not read installation permissions, scanning comments without the permission precheck", "error", err)
		return nil
	}
	return grants
}

// trimURLAndSplit removes extraneous information from the |url| and splits it into segments.
// This is typically 3 segments: host, owner, and name/ID; however, Gists have some edge cases.
//
// Examples:
// - "https://github.com/trufflesecurity/trufflehog" => ["github.com", "trufflesecurity", "trufflehog"]
// - "https://gist.github.com/nat/5fdbb7f945d121f197fb074578e53948" => ["gist.github.com", "nat", "5fdbb7f945d121f197fb074578e53948"]
// - "https://gist.github.com/ff0e5e8dc8ec22f7a25ddfc3492d3451.git" => ["gist.github.com", "ff0e5e8dc8ec22f7a25ddfc3492d3451"]
// - "https://github.company.org/gist/nat/5fdbb7f945d121f197fb074578e53948.git" => ["github.company.org", "gist", "nat", "5fdbb7f945d121f197fb074578e53948"]
func getRepoURLParts(repoURLString string) (string, []string, error) {
	// Support ssh and https URLs.
	repoURL, err := git.GitURLParse(repoURLString)
	if err != nil {
		return "", nil, err
	}

	// Remove the user information.
	// e.g., `[email protected]` -> `github.com`
	if repoURL.User != nil {
		repoURL.User = nil
	}

	// Use Host and Path directly instead of reconstructing via String().
	// This preserves special characters like trailing hyphens in repo names
	// that might be lost during URL reconstruction.
	// See: https://github.com/trufflesecurity/trufflehog/issues/4679
	host := repoURL.Host
	path := strings.TrimPrefix(repoURL.Path, "/")
	path = strings.TrimSuffix(path, ".git")

	urlString := repoURL.String()
	urlParts := append([]string{host}, strings.Split(path, "/")...)

	// Validate
	switch len(urlParts) {
	case 2:
		// gist.github.com/<gist_id>
		if !strings.EqualFold(urlParts[0], "gist.github.com") {
			err = fmt.Errorf("failed to parse repository or gist URL (%s): 2 path segments are only expected if the host is 'gist.github.com' ('gist.github.com', '<gist_id>')", urlString)
		}
	case 3:
		// github.com/<user>/repo>
		// gist.github.com/<user>/<gist_id>
		// github.company.org/<user>/repo>
		// github.company.org/gist/<gist_id>
	case 4:
		// github.company.org/gist/<user/<id>
		if !strings.EqualFold(urlParts[1], "gist") || (strings.EqualFold(urlParts[0], "github.com") && strings.EqualFold(urlParts[1], "gist")) {
			err = fmt.Errorf("failed to parse repository or gist URL (%s): 4 path segments are only expected if the host isn't 'github.com' and the path starts with 'gist' ('github.example.com', 'gist', '<owner>', '<gist_id>')", urlString)
		}
	default:
		err = fmt.Errorf("invalid repository or gist URL (%s): length of URL segments should be between 2 and 4, not %d (%v)", urlString, len(urlParts), urlParts)
	}

	if err != nil {
		return "", nil, err
	}
	return urlString, urlParts, nil
}

// commentPager returns the pager every comment walk for one repo shares. Rate
// limits are handled by waiting and retrying the same request, and each wait
// is also reported to the chunk reporter. Transient failures are retried a
// bounded number of times; see fetchWithRetry.
func (s *Source) commentPager(ctx context.Context, apiClient *github.Client, reporter sources.ChunkReporter) pager {
	wait := s.commentRetryWait
	if wait == nil {
		wait = func(ctx context.Context, d time.Duration) bool {
			_, canceled := sleepWithContext(ctx, d)
			return !canceled
		}
	}
	return pager{
		client:  apiClient,
		perPage: defaultPagination,
		retry: func(err error) bool {
			return s.handleRateLimitWithChunkReporter(ctx, reporter, err)
		},
		wait: wait,
	}
}

// processGistComments scans a gist's comments. The endpoint has no `since` or
// sort parameters, so the comments timeframe is applied to each page as it
// arrives, and a capped walk has no way to continue: it is reported as an
// error rather than ending silently.
func (s *Source) processGistComments(ctx context.Context, apiClient *github.Client, gistURL string, urlParts []string, repoInfo repoInfo, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
	ctx.Logger().V(2).Info("Scanning GitHub Gist comments")

	gistID := extractGistID(urlParts)
	p := s.commentPager(ctx, apiClient, reporter)
	p.listing = "gist comments"
	outcome, err := walkPages(ctx, p,
		func(ctx context.Context) ([]*github.GistComment, *github.Response, error) {
			return apiClient.Gists.ListComments(ctx, gistID, &github.ListOptions{PerPage: defaultPagination})
		},
		func(comments []*github.GistComment) error {
			return s.chunkGistComments(ctx, gistURL, repoInfo, comments, reporter, cutoffTime)
		},
	)
	if err != nil {
		return fmt.Errorf("gist comments: %w", &walkError{items: outcome.items, err: err})
	}
	if outcome.capped {
		return fmt.Errorf("gist comments: %w", &walkError{
			items: outcome.items,
			err:   errors.New("GitHub stopped paginating and this endpoint has no since filter to continue from"),
		})
	}
	return nil
}

func extractGistID(urlParts []string) string {
	return urlParts[len(urlParts)-1]
}

// isGistUrl returns true if the URL path is of a gist
func isGistUrl(urlParts []string) bool {
	if len(urlParts) == 0 {
		return false
	}

	// standard github gists: gist.github.com/user/abc123
	if strings.EqualFold(urlParts[0], "gist.github.com") {
		return true
	}

	// github enterprise: any 3 or 4 parts url with 'gist'
	if (len(urlParts) == 3 || len(urlParts) == 4) && slices.Contains(urlParts, "gist") {
		// enterprise.company.com/gist/gist-id
		// gist.company.com/gist/gist-id
		// gist.company.com/path/gist/gist-id
		return true
	}

	return false
}

func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo repoInfo, comments []*github.GistComment, reporter sources.ChunkReporter, cutoffTime *time.Time) error {
	for _, comment := range comments {
		// Gist comments come back oldest first, so an old comment says nothing
		// about the ones after it. Skip it and keep going.
		if cutoffTime != nil && comment.GetCreatedAt().Before(*cutoffTime) {
			continue
		}

		// Create chunk and send it to the channel.
		chunk := sources.Chunk{
			SourceName: s.name,
			SourceID:   s.SourceID(),
			SourceType: s.Type(),
			JobID:      s.JobID(),
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Github{
					Github: &source_metadatapb.Github{
						Link:       sanitizer.UTF8(comment.GetURL()),
						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
						Repository: sanitizer.UTF8(gistURL),
						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
						Visibility: gistInfo.visibility,
					},
				},
			},
			Data:         []byte(sanitizer.UTF8(comment.GetBody())),
			SourceVerify: s.verify,
		}

		if err := reporter.ChunkOk(ctx, chunk); err != nil {
			return err
		}
	}
	return nil
}

// Repo listings sort by update time, oldest first, so a capped walk can
// continue from the last update time it saw. See walkUpdatedSince.
const (
	sortByUpdated = "updated"
	sortAscending = "asc"
	// stateAll lists open and closed issues and pull requests.
	stateAll = "all"
)

// commentPhase is one listing in a repo's comment scan. run returns how many
// chunks it emitted. startLog, when set, is logged at V(2) before the phase
// runs.
//
// skipped and narrowed carry what the installation's grants cost the phase
// (see installationGrants). skipped, when set, is why the listing can't
// succeed; the phase reports it in place of running. narrowed holds one error
// per kind of item GitHub will leave out of a listing that still runs, each
// already prefixed with what's missing, and is reported after the run.
type commentPhase struct {
	name     string
	startLog string
	run      func() (int, error)
	skipped  error
	narrowed []error
}

// processIssueandPRsWithCommentsREST scans a repo's issues, pull requests, and
// their comments through the REST API.
//
// The phases are independent: a failure in one is collected and the rest
// still run, so one capped or forbidden endpoint can't hide the others. All
// failures come back joined into one error, which scanRepo splits back apart
// so each failed phase is its own ChunkErr (see reportCommentErrors). Chunks
// emitted before a failure are kept.
//
// grants, when known, shapes the phases. A mismatch between the comment
// settings and what the installation grants is a configuration error the
// customer can fix, so it's always reported: every listing skipped because it
// can't succeed, and every kind of item GitHub will silently leave out of a
// listing that does run, gets exactly one permissionError. Items no setting
// asked for don't count. Skipped listings make no request.
//
// The "Scanning issues" and "Scanning pull requests" V(2) lines are the only
// log signal for scans on the unit path, whose comment errors go to the scan
// details UI instead of the logs, so they keep their wording. The summary
// line at the end records how far each phase got.
func (s *Source) processIssueandPRsWithCommentsREST(
	ctx context.Context, apiClient *github.Client, repoInfo repoInfo,
	grants *installationGrants, reporter sources.ChunkReporter, cutoffTime *time.Time,
) error {
	p := s.commentPager(ctx, apiClient, reporter)
	var since time.Time
	if cutoffTime != nil {
		since = *cutoffTime
	}

	var installationID int64
	if grants != nil {
		installationID = grants.installationID
	}
	issuesOK, prsOK := grants.issuesGranted(), grants.pullRequestsGranted()
	narrowed := func(missing, listing string, needs permission) error {
		return fmt.Errorf("%s: %w", missing, newNarrowedListingError(installationID, listing, []permission{needs}))
	}
	// noPRs is set when /pulls ran and found nothing. Its state=all listing
	// then shows the repo has no pull requests, so it has no review comments
	// either. A 404 or 410 means pull requests are off, which is different:
	// /pulls/comments still serves review comments from before that.
	var noPRs bool
	pullsPhase := func(startLog string) commentPhase {
		return commentPhase{name: "pull requests", startLog: startLog, run: func() (int, error) {
			emitted, err := s.processPRs(ctx, p, repoInfo, reporter)
			if isGitHubListingDisabled(err) {
				ctx.Logger().V(2).Info("pull requests listing disabled for repo, skipping")
				return emitted, nil
			}
			noPRs = err == nil && emitted == 0
			return emitted, err
		}}
	}

	// /issues returns issues and pull requests together. When issue comments
	// are enabled, that one walk also emits PR bodies, so /pulls is only
	// walked for configs that enable PR comments alone, or when the
	// installation can list pull requests but not issues.
	var phases []commentPhase
	if s.includeIssueComments {
		bodies := commentPhase{name: "issues", startLog: "Scanning issues", run: func() (int, error) {
			return s.processIssues(ctx, p, repoInfo, reporter, s.includePRComments)
		}}
		needs := []permission{issuesRead}
		if s.includePRComments {
			bodies.name = "issues and pull requests"
			needs = append(needs, pullRequestsRead)
		}
		switch {
		case issuesOK:
			if s.includePRComments && !prsOK {
				bodies.narrowed = append(bodies.narrowed, narrowed("pull request bodies", "/issues", pullRequestsRead))
			}
			phases = append(phases, bodies)
		case s.includePRComments && prsOK:
			// /issues is refused without Issues (read), but the PR bodies it
			// would have carried are still reachable through /pulls.
			phases = append(phases,
				commentPhase{name: "issues", skipped: newSkippedListingError(installationID, []permission{issuesRead})},
				pullsPhase(""))
		default:
			bodies.skipped = newSkippedListingError(installationID, needs)
			phases = append(phases, bodies)
		}

		// /issues/comments carries comments on issues and on pull requests,
		// each kind only when its permission is granted. PR conversation
		// comments are scanned only when PR comments are enabled, so their
		// absence otherwise isn't a mismatch.
		comments := commentPhase{name: "issue comments", run: func() (int, error) {
			return s.processIssueComments(ctx, p, repoInfo, reporter, since)
		}}
		switch {
		case !issuesOK && !prsOK:
			comments.skipped = newSkippedListingError(installationID, needs)
		case !issuesOK:
			comments.narrowed = append(comments.narrowed, narrowed("issue comments", "/issues/comments", issuesRead))
		case s.includePRComments && !prsOK:
			comments.narrowed = append(comments.narrowed, narrowed("pull request conversation comments", "/issues/comments", pullRequestsRead))
		}
		phases = append(phases, comments)
	}
	if s.includePRComments {
		var prsSkipped error
		if !prsOK {
			prsSkipped = newSkippedListingError(installationID, []permission{pullRequestsRead})
		}
		prStartLog := "Scanning pull requests"
		if !s.includeIssueComments {
			pulls := pullsPhase(prStartLog)
			pulls.skipped = prsSkipped
			phases = append(phases, pulls)
			prStartLog = ""
		}
		phases = append(phases, commentPhase{name: "pull request comments", startLog: prStartLog, skipped: prsSkipped, run: func() (int, error) {
			if noPRs {
				ctx.Logger().V(2).Info("no pull requests in repo, skipping pull request comments")
				return 0, nil
			}
			return s.processPRComments(ctx, p, repoInfo, reporter, since)
		}})
	}

	var (
		errs        []error
		failed      []string
		skipped     []string
		narrowedOut []string
		summary     []any
	)
	for _, phase := range phases {
		if err := ctx.Err(); err != nil {
			errs = append(errs, err)
			break
		}
		if phase.skipped != nil {
			skipped = append(skipped, phase.name)
			errs = append(errs, fmt.Errorf("%s: %w", phase.name, phase.skipped))
			continue
		}
		if phase.startLog != "" {
			ctx.Logger().V(2).Info(phase.startLog)
		}
		emitted, err := phase.run()
		summary = append(summary, strings.ReplaceAll(phase.name, " ", "_"), emitted)
		if err != nil {
			failed = append(failed, phase.name)
			errs = append(errs, fmt.Errorf("%s: %w", phase.name, err))
		}
		if len(phase.narrowed) > 0 {
			narrowedOut = append(narrowedOut, phase.name)
			errs = append(errs, phase.narrowed...)
		}
	}
	ctx.Logger().V(2).Info("finished scanning comments", append(summary,
		"failed_phases", failed, "skipped_phases", skipped, "narrowed_phases", narrowedOut)...)

	return errors.Join(errs...)
}

func (s *Source) processRepoIssueandPRsWithCommentsGraphql(
	ctx context.Context, graphqlClient *githubv4.Client, repoInfo repoInfo,
	reporter sources.ChunkReporter, cutoffTime *time.Time,
) error {
	if s.includeIssueComments {
		ctx.Logger().V(2).Info("Scanning issues")
		if err := s.processIssuesWithComments(ctx, graphqlClient, repoInfo, reporter, cutoffTime); err != nil {
			return err
		}
	}

	if s.includePRComments {
		ctx.Logger().V(2).Info("Scanning pull requests")
		if err := s.processPRWithComments(ctx, graphqlClient, repoInfo, reporter, cutoffTime); err != nil {
			return err
		}

		if err := s.processReviewThreads(ctx, graphqlClient, repoInfo, reporter, cutoffTime); err != nil {
			return err
		}
	}

	return nil
}

// processIssues walks /issues and emits issue bodies, plus pull request
// bodies when includePRs is set. The listing always contains both, because
// GitHub treats every pull request as an issue. Bodies are scanned in full
// regardless of the comments timeframe.
func (s *Source) processIssues(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, includePRs bool) (int, error) {
	emitted := 0
	_, err := walkUpdatedSince(ctx, p, sinceWalk[*github.Issue]{
		name: "issues",
		list: listIssuesSince(p, repoInfo),
		key:  issueKey,
		visit: func(issues []*github.Issue) error {
			n, err := s.chunkIssues(ctx, repoInfo, issues, reporter, func(issue *github.Issue) bool {
				return includePRs || !issue.IsPullRequest()
			})
			emitted += n
			return err
		},
	})
	return emitted, err
}

// listIssuesSince returns the /issues listing, issues and pull requests
// together, for walkUpdatedSince.
func listIssuesSince(p pager, repoInfo repoInfo) func(context.Context, time.Time) ([]*github.Issue, *github.Response, error) {
	return func(ctx context.Context, since time.Time) ([]*github.Issue, *github.Response, error) {
		return p.client.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, &github.IssueListByRepoOptions{
			State:       stateAll,
			Sort:        sortByUpdated,
			Direction:   sortAscending,
			Since:       since,
			ListOptions: github.ListOptions{PerPage: p.perPage},
		})
	}
}

func issueKey(issue *github.Issue) (int64, time.Time) {
	return issue.GetID(), issue.GetUpdatedAt().Time
}

// chunkIssues emits a chunk for each issue that keep accepts, and returns how
// many it emitted. Pull requests from /issues produce the same chunk as
// chunkPullRequests: same link, author, timestamp, and title and body.
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, reporter sources.ChunkReporter, keep func(*github.Issue) bool) (int, error) {
	emitted := 0
	for _, issue := range issues {
		if !keep(issue) {
			continue
		}

		// Create chunk and send it to the channel.
		chunk := sources.Chunk{
			SourceName: s.name,
			SourceID:   s.SourceID(),
			JobID:      s.JobID(),
			SourceType: s.Type(),
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Github{
					Github: &source_metadatapb.Github{
						Link:       sanitizer.UTF8(issue.GetHTMLURL()),
						Username:   sanitizer.UTF8(issue.GetUser().GetLogin()),
						Email:      sanitizer.UTF8(issue.GetUser().GetEmail()),
						Repository: sanitizer.UTF8(repoInfo.fullName),
						Timestamp:  sanitizer.UTF8(issue.GetCreatedAt().String()),
						Visibility: repoInfo.visibility,
					},
				},
			},
			Data:         []byte(sanitizer.UTF8(issue.GetTitle() + "\n" + issue.GetBody())),
			SourceVerify: s.verify,
		}

		if err := reporter.ChunkOk(ctx, chunk); err != nil {
			return emitted, err
		}
		emitted++
	}
	return emitted, nil
}

// processIssueComments walks every issue comment in the repo updated at or
// after since. /issues/comments also returns comments on pull requests, since
// GitHub treats every pull request as an issue. The endpoint stops at 300
// pages, so the walk continues past that cap in since windows.
//
// When both issues and pull requests are off, GitHub answers 404 here. That
// is only treated as empty when the repo also reports issues off, so an
// unexpected 404 is still reported.
func (s *Source) processIssueComments(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, since time.Time) (int, error) {
	emitted, err := walkUpdatedSince(ctx, p, sinceWalk[*github.IssueComment]{
		name:  "issue comments",
		since: since,
		list: func(ctx context.Context, since time.Time) ([]*github.IssueComment, *github.Response, error) {
			opts := &github.IssueListCommentsOptions{
				Sort:        github.String(sortByUpdated),
				Direction:   github.String(sortAscending),
				ListOptions: github.ListOptions{PerPage: p.perPage},
			}
			if !since.IsZero() {
				opts.Since = &since
			}
			// Issue number 0 lists comments across the whole repo.
			return p.client.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, 0, opts)
		},
		key: func(comment *github.IssueComment) (int64, time.Time) {
			return comment.GetID(), comment.GetUpdatedAt().Time
		},
		visit: func(comments []*github.IssueComment) error {
			return s.chunkIssueComments(ctx, repoInfo, comments, reporter)
		},
	})
	if err != nil && repoInfo.hasNoIssues && isGitHubListingDisabled(err) {
		ctx.Logger().V(2).Info("issue comments listing disabled for repo, skipping")
		return emitted, nil
	}
	return emitted, err
}

// chunkIssueComments emits a chunk for each comment. The comments timeframe
// is already applied by the API through `since`.
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, reporter sources.ChunkReporter) error {
	for _, comment := range comments {
		// Create chunk and send it to the channel.
		chunk := sources.Chunk{
			SourceName: s.name,
			SourceID:   s.SourceID(),
			JobID:      s.JobID(),
			SourceType: s.Type(),
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Github{
					Github: &source_metadatapb.Github{
						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
						Repository: sanitizer.UTF8(repoInfo.fullName),
						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
						Visibility: repoInfo.visibility,
					},
				},
			},
			Data:         []byte(sanitizer.UTF8(comment.GetBody())),
			SourceVerify: s.verify,
		}

		if err := reporter.ChunkOk(ctx, chunk); err != nil {
			return err
		}
	}
	return nil
}

// processPRs emits pull request bodies for configs that enable PR comments
// but not issue comments; otherwise the /issues walk emits them. /pulls has
// no `since` filter, so if GitHub caps it, the walk falls back to a windowed
// /issues walk that emits only pull requests, skipping the ones /pulls
// already emitted. The common case never pages through issues.
func (s *Source) processPRs(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter) (int, error) {
	emitted := 0
	emittedNumbers := map[int]struct{}{}
	p.listing = "pull requests"
	outcome, err := walkPages(ctx, p,
		func(ctx context.Context) ([]*github.PullRequest, *github.Response, error) {
			return p.client.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, &github.PullRequestListOptions{
				State:       stateAll,
				Sort:        sortByUpdated,
				Direction:   sortAscending,
				ListOptions: github.ListOptions{PerPage: p.perPage},
			})
		},
		func(prs []*github.PullRequest) error {
			if err := s.chunkPullRequests(ctx, repoInfo, prs, reporter); err != nil {
				return err
			}
			for _, pr := range prs {
				emittedNumbers[pr.GetNumber()] = struct{}{}
			}
			emitted += len(prs)
			return nil
		},
	)
	if err != nil {
		return emitted, &walkError{items: outcome.items, err: err}
	}
	if !outcome.capped {
		return emitted, nil
	}

	ctx.Logger().V(2).Info("pull request listing capped, continuing from the issue listing", "pull_requests_so_far", emitted)
	_, err = walkUpdatedSince(ctx, p, sinceWalk[*github.Issue]{
		name: "pull requests from issues",
		list: listIssuesSince(p, repoInfo),
		key:  issueKey,
		visit: func(issues []*github.Issue) error {
			n, err := s.chunkIssues(ctx, repoInfo, issues, reporter, func(issue *github.Issue) bool {
				if !issue.IsPullRequest() {
					return false
				}
				_, done := emittedNumbers[issue.GetNumber()]
				return !done
			})
			emitted += n
			return err
		},
	})
	return emitted, err
}

// processPRComments walks every pull request review comment (the inline
// comments on diffs) in the repo updated at or after since. Discussion
// comments on pull requests come from /issues/comments instead. No cap has
// been observed on this endpoint, but it gets the same since windows as
// /issues/comments in case one exists.
func (s *Source) processPRComments(ctx context.Context, p pager, repoInfo repoInfo, reporter sources.ChunkReporter, since time.Time) (int, error) {
	return walkUpdatedSince(ctx, p, sinceWalk[*github.PullRequestComment]{
		name:  "pull request comments",
		since: since,
		list: func(ctx context.Context, since time.Time) ([]*github.PullRequestComment, *github.Response, error) {
			// Pull request number 0 lists review comments across the whole repo.
			return p.client.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, 0, &github.PullRequestListCommentsOptions{
				Sort:        sortByUpdated,
				Direction:   sortAscending,
				Since:       since,
				ListOptions: github.ListOptions{PerPage: p.perPage},
			})
		},
		key: func(comment *github.PullRequestComment) (int64, time.Time) {
			return comment.GetID(), comment.GetUpdatedAt().Time
		},
		visit: func(comments []*github.PullRequestComment) error {
			return s.chunkPullRequestComments(ctx, repoInfo, comments, reporter)
		},
	})
}

func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, reporter sources.ChunkReporter) error {
	for _, pr := range prs {
		// Create chunk and send it to the channel.
		chunk := sources.Chunk{
			SourceName: s.name,
			SourceID:   s.SourceID(),
			SourceType: s.Type(),
			JobID:      s.JobID(),
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Github{
					Github: &source_metadatapb.Github{
						Link:       sanitizer.UTF8(pr.GetHTMLURL()),
						Username:   sanitizer.UTF8(pr.GetUser().GetLogin()),
						Email:      sanitizer.UTF8(pr.GetUser().GetEmail()),
						Repository: sanitizer.UTF8(repoInfo.fullName),
						Timestamp:  sanitizer.UTF8(pr.GetCreatedAt().String()),
						Visibility: repoInfo.visibility,
					},
				},
			},
			Data:         []byte(sanitizer.UTF8(pr.GetTitle() + "\n" + pr.GetBody())),
			SourceVerify: s.verify,
		}

		if err := reporter.ChunkOk(ctx, chunk); err != nil {
			return err
		}
	}
	return nil
}

// chunkPullRequestComments emits a chunk for each review comment. The
// comments timeframe is already applied by the API through `since`.
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, reporter sources.ChunkReporter) error {
	for _, comment := range comments {
		// Create chunk and send it to the channel.
		chunk := sources.Chunk{
			SourceName: s.name,
			SourceID:   s.SourceID(),
			JobID:      s.JobID(),
			SourceType: s.Type(),
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Github{
					Github: &source_metadatapb.Github{
						Link:       sanitizer.UTF8(comment.GetHTMLURL()),
						Username:   sanitizer.UTF8(comment.GetUser().GetLogin()),
						Email:      sanitizer.UTF8(comment.GetUser().GetEmail()),
						Repository: sanitizer.UTF8(repoInfo.fullName),
						Timestamp:  sanitizer.UTF8(comment.GetCreatedAt().String()),
						Visibility: repoInfo.visibility,
					},
				},
			},
			Data:         []byte(sanitizer.UTF8(comment.GetBody())),
			SourceVerify: s.verify,
		}

		if err := reporter.ChunkOk(ctx, chunk); err != nil {
			return err
		}
	}
	return nil
}

func (s *Source) scanTargets(ctx context.Context, targets []sources.ChunkingTarget, reporter sources.ChunkReporter) []error {
	var errs []error
	for _, tgt := range targets {
		if err := s.scanTarget(ctx, tgt, reporter); err != nil {
			ctx.Logger().Error(err, "error scanning target")
			errs = append(errs, &sources.TargetedScanError{Err: err, SecretID: tgt.SecretID})
		}
	}

	return errs
}

func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget, reporter sources.ChunkReporter) error {
	metaType, ok := target.QueryCriteria.GetData().(*source_metadatapb.MetaData_Github)
	if !ok {
		return fmt.Errorf("unable to cast metadata type for targeted scan")
	}
	meta := metaType.Github

	chunkSkel := sources.Chunk{
		SourceType: s.Type(),
		SourceName: s.name,
		SourceID:   s.SourceID(),
		JobID:      s.JobID(),
		SecretID:   target.SecretID,
		SourceMetadata: &source_metadatapb.MetaData{
			Data: &source_metadatapb.MetaData_Github{Github: meta},
		},
		SourceVerify: s.verify,
	}

	u, err := url.Parse(meta.GetLink())
	if err != nil {
		return fmt.Errorf("unable to parse GitHub URL: %w", err)
	}

	// The owner is the second segment and the repo is the third segment of the path.
	// Ex: https://github.com/owner/repo/.....
	segments := strings.Split(u.Path, "/")
	if len(segments) < 3 {
		return fmt.Errorf("invalid GitHub URL")
	}

	repoURL, err := s.repoURLFromTargetMetadata(meta)
	if err != nil {
		return err
	}
	targetIsWiki := isWikiLink(meta.GetLink())
	if targetIsWiki {
		repoURL, err = repoCloneURLForTargetEndpoint(s.conn.GetEndpoint(), repoURL)
		if err != nil {
			return err
		}
	}
	if connector, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations {
		if err := s.mapReposToInstallations(ctx, connector, []string{repoURL}); err != nil {
			return err
		}
	}
	if targetIsWiki {
		wikiURL, ok := wikiCloneURLForRepo(repoURL)
		if !ok {
			return fmt.Errorf("could not derive wiki clone URL from target repository %q", repoURL)
		}
		return s.scanWikiTarget(ctx, wikiURL, meta, &chunkSkel, reporter)
	}
	apiClient, err := s.connector.APIClientForRepo(repoURL)
	if err != nil {
		return err
	}

	if meta.GetFile() == "" && meta.GetCommit() != "" {
		ctx := context.WithValues(ctx, "commit_hash", meta.GetCommit())
		ctx.Logger().V(4).Info("secret metadata has no file; scanning commit metadata instead")

		return s.scanCommitMetadata(ctx, apiClient, segments[1], segments[2], meta, &chunkSkel, reporter)
	}

	// else try downloading the file content to scan
	readCloser, resp, err := apiClient.Repositories.DownloadContents(
		ctx,
		segments[1],
		segments[2],
		meta.GetFile(),
		&github.RepositoryContentGetOptions{Ref: meta.GetCommit()})
	// As of this writing, if the returned readCloser is not nil, it's just the Body of the returned github.Response, so
	// there's no need to independently close it.
	if resp != nil && resp.Body != nil {
		defer func() { _ = resp.Body.Close() }()
	}
	if err != nil {
		// DownloadContents locates the file by listing its parent directory,
		// and the contents API caps listings at 1000 entries, so it can fail
		// for files that exist. Retry with an exact-path lookup, which also
		// serves as the existence probe when it fails.
		retryCloser, retryResp, retryErr := s.downloadContentsByPath(
			ctx, apiClient, segments[1], segments[2], meta.GetFile(), meta.GetCommit())
		if retryResp != nil && retryResp.Response != nil && retryResp.Body != nil {
			defer func() { _ = retryResp.Body.Close() }()
		}
		if retryErr != nil {
			wrapped := fmt.Errorf("could not download file for scan: %w", retryErr)
			// The exact-path lookup 404ing is authoritative for the path, but
			// GitHub also returns 404 for existing resources the credentials
			// cannot see, so only classify the target as gone when the
			// repository itself is still reachable with the same client.
			if retryResp != nil && retryResp.Response != nil &&
				retryResp.StatusCode == http.StatusNotFound &&
				s.repoReachable(ctx, apiClient, segments[1], segments[2]) {
				return &sources.TargetNotFoundError{Err: wrapped}
			}
			return wrapped
		}
		fileCtx := context.WithValues(ctx, "path", meta.GetFile())
		return handlers.HandleFile(fileCtx, retryCloser, &chunkSkel, reporter)
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected HTTP response status when trying to download file for scan: %v", resp.Status)
	}

	fileCtx := context.WithValues(ctx, "path", meta.GetFile())
	return handlers.HandleFile(fileCtx, readCloser, &chunkSkel, reporter)
}

// downloadContentsByPath fetches a file the same way DownloadContents does,
// but resolves it with an exact-path contents lookup instead of searching a
// directory listing, so it is immune to the 1000-entry listing cap. On
// failure the returned response is always the exact-path lookup's, whose 404
// is an authoritative statement about the path at that ref; a failure of the
// download itself must not be mistaken for the target being gone, because
// the lookup just proved it exists.
func (s *Source) downloadContentsByPath(ctx context.Context, apiClient *github.Client, owner, repo, filePath, ref string) (io.ReadCloser, *github.Response, error) {
	fileContent, _, resp, err := apiClient.Repositories.GetContents(
		ctx, owner, repo, filePath, &github.RepositoryContentGetOptions{Ref: ref})
	if err != nil {
		return nil, resp, err
	}
	if fileContent == nil || fileContent.GetDownloadURL() == "" {
		return nil, resp, fmt.Errorf("no download link found for %s", filePath)
	}
	dlReq, err := http.NewRequestWithContext(ctx, http.MethodGet, fileContent.GetDownloadURL(), nil)
	if err != nil {
		return nil, resp, err
	}
	dlResp, err := apiClient.Client().Do(dlReq)
	if err != nil {
		return nil, resp, err
	}
	if dlResp.StatusCode != http.StatusOK {
		_ = dlResp.Body.Close()
		return nil, resp, fmt.Errorf("unexpected HTTP response status when trying to download file for scan: %v", dlResp.Status)
	}
	return dlResp.Body, &github.Response{Response: dlResp}, nil
}

// repoReachable reports whether the repository is still visible with the
// same client.
func (s *Source) repoReachable(ctx context.Context, apiClient *github.Client, owner, repo string) bool {
	_, resp, err := apiClient.Repositories.Get(ctx, owner, repo)
	return err == nil && resp != nil && resp.StatusCode == http.StatusOK
}

func (s *Source) scanWikiTarget(ctx context.Context, wikiURL string, meta *source_metadatapb.Github, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) error {
	if meta.GetCommit() == "" {
		return fmt.Errorf("wiki target has no commit")
	}

	path, repo, err := s.cloneRepo(ctx, wikiURL)
	if err != nil {
		return fmt.Errorf("could not clone wiki for targeted scan: %w", err)
	}
	if !s.conn.GetPrintLegacyJson() {
		if strings.HasPrefix(path, filepath.Join(os.TempDir(), "trufflehog")) || (!s.conn.NoCleanup && s.conn.GetClonePath() != "") {
			defer func() { _ = os.RemoveAll(path) }()
		}
	}

	commit, err := repo.CommitObject(plumbing.NewHash(meta.GetCommit()))
	if err != nil {
		return fmt.Errorf("could not fetch wiki commit for targeted scan: %w", err)
	}

	if meta.GetFile() == "" {
		var sb strings.Builder
		sb.WriteString(commit.Author.Email)
		sb.WriteString("\n")
		sb.WriteString(commit.Committer.Email)
		sb.WriteString("\n")
		sb.WriteString(commit.Message)
		return handlers.HandleFile(ctx, io.NopCloser(strings.NewReader(sb.String())), chunkSkel, reporter)
	}

	file, err := commit.File(meta.GetFile())
	if err != nil {
		return fmt.Errorf("could not find wiki file for targeted scan: %w", err)
	}
	readCloser, err := file.Reader()
	if err != nil {
		return fmt.Errorf("could not read wiki file for targeted scan: %w", err)
	}
	defer func() { _ = readCloser.Close() }()

	fileCtx := context.WithValues(ctx, "path", meta.GetFile())
	return handlers.HandleFile(fileCtx, readCloser, chunkSkel, reporter)
}

func (s *Source) scanCommitMetadata(ctx context.Context, apiClient *github.Client, owner, repo string, meta *source_metadatapb.Github, chunkSkel *sources.Chunk, reporter sources.ChunkReporter) error {
	// fetch the commit
	commit, resp, err := apiClient.Repositories.GetCommit(ctx, owner, repo, meta.GetCommit(), nil)
	if resp != nil && resp.Body != nil {
		defer func() { _ = resp.Body.Close() }()
	}
	if err != nil {
		return fmt.Errorf("could not fetch commit for metadata scan: %w", err)
	}

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected HTTP response status when fetching commit: %v", resp.Status)
	}

	// create the string with the exact format we use in Git.ScanCommits()
	// author email + "\n" + committer + "\n" + commit message
	var sb strings.Builder

	sb.WriteString(commit.GetCommit().Author.GetEmail())
	sb.WriteString("\n")
	sb.WriteString(commit.GetCommitter().GetEmail())
	sb.WriteString("\n")
	sb.WriteString(commit.GetCommit().GetMessage())

	content := strings.NewReader(sb.String())
	return handlers.HandleFile(ctx, io.NopCloser(content), chunkSkel, reporter)
}

func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
	repoURL, kind := unit.SourceUnitID()
	ctx = context.WithValue(ctx, "repo", safeRepoURL(repoURL))

	// If the unit has an installation ID, use it. Fetching it is slow,
	// specifically it stalls large multi-org scans on rate limits (INT-790).

	// [CG] This is pretty ugly; if you can clean it up please do (I failed).
	if ac, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations && kind == "repo" {
		if ru, ok := unit.(RepoUnit); ok && ru.InstallationID > 0 {
			ac.setRepoInstallationForRepoName(repoURL, ru.Name, ru.InstallationID)
		} else if err := s.mapReposToInstallations(ctx, ac, []string{repoURL}); err != nil {
			return err
		}
	}

	// ChunkUnit is not guaranteed to be called from Enumerate, so we must
	// check and fetch the repoInfoCache for this repo.
	repoURL, err := s.ensureRepoInfoCache(ctx, repoURL, &chunkErrorReporter{reporter: reporter})
	if err != nil {
		return err
	}
	return s.scanRepo(ctx, repoURL, reporter)
}