Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Clean up GitHub source logging for per-source debug (#5457)
Kashif Khan0301dc36bd67
diff --git a/pkg/engine/scan.go b/pkg/engine/scan.goindex e9ed7f3c3..85e1e0184 100644--- a/pkg/engine/scan.go+++ b/pkg/engine/scan.go@@ -16,12 +16,21 @@ func (e *Engine) ScanConfig(ctx context.Context, configuredSources ...sources.Cofor _, configuredSource := range configuredSources {sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, configuredSource.Name, configuredSource.SourceType())+ // Add the source fields here, before Init, so lines logged during Init+ // say which source and job they came from too. The source manager adds+ // the same fields later for the scan, but skips any already set here.+ sourceCtx := context.WithValues(ctx,+ "source_name", configuredSource.Name,+ "source_type", configuredSource.SourceType().String(),+ "source_id", sourceID,+ "job_id", jobID,+ )+// A source with debug on gets a context that writes all of its debug// logs. Both Init and the scan use it, so every log from this source is// covered. Other sources keep the global log level.- sourceCtx := ctxif configuredSource.Debug {- sourceCtx = context.WithLogLevel(ctx, log.DebugLevel)+ sourceCtx = context.WithLogLevel(sourceCtx, log.DebugLevel)}source, err := configuredSource.Init(sourceCtx, sourceID, jobID)diff --git a/pkg/engine/scan_test.go b/pkg/engine/scan_test.goindex cc9ec2094..f21d2f405 100644--- a/pkg/engine/scan_test.go+++ b/pkg/engine/scan_test.go@@ -82,4 +82,21 @@ func TestScanConfig_DebugOnlyForOneSource(t *testing.T) {assert.Equal(t, 1, countLines("chunks debug", "debug"))assert.Zero(t, countLines("init debug", "normal"))assert.Zero(t, countLines("chunks debug", "normal"))++ // Lines from Init and from the scan both carry the source fields, and+ // each field shows up once even though the source manager adds them too.+ for _, message := range []string{"init info", "chunks info"} {+ for _, entry := range logs.FilterMessage(message).FilterField(zap.String("source", "debug")).All() {+ for _, key := range []string{"source_name", "source_type", "source_id", "job_id"} {+ count := 0+ for _, field := range entry.Context {+ if field.Key == key {+ count+++ }+ }+ assert.Equal(t, 1, count, "%s on %q", key, message)+ }+ assert.Equal(t, "debug", entry.ContextMap()["source_name"], message)+ }+ }}diff --git a/pkg/sources/github/comments_test.go b/pkg/sources/github/comments_test.goindex c7f1ff7af..741e4e286 100644--- a/pkg/sources/github/comments_test.go+++ b/pkg/sources/github/comments_test.go@@ -630,8 +630,9 @@ func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {includePRComments: true,commentRetryWait: func(context.Context, time.Duration) bool { return true },}+ // retries are logged at V(4), with other API decisionsvar logs bytes.Buffer- logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(2)))+ logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(4)))ctx := context.WithLogger(context.Background(), logger)err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, nil, &collectingReporter{}, nil)diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.goindex 9e99e4e1f..bd4932511 100644--- a/pkg/sources/github/connector.go+++ b/pkg/sources/github/connector.go@@ -117,7 +117,7 @@ func newConnector(ctx context.Context, source *Source) (Connector, error) {switch cred := source.conn.GetCredential().(type) {case *sourcespb.GitHub_GithubApp:log.RedactGlobally(cred.GithubApp.GetPrivateKey())- return NewAppConnector(apiEndpoint, cred.GithubApp, source.conn.GetScanAllInstallations())+ return NewAppConnector(ctx, apiEndpoint, cred.GithubApp, source.conn.GetScanAllInstallations())case *sourcespb.GitHub_BasicAuth:log.RedactGlobally(cred.BasicAuth.GetPassword())return NewBasicAuthConnector(ctx, apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth)diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.goindex 17e0ec0ef..60cf8753e 100644--- a/pkg/sources/github/connector_app.go+++ b/pkg/sources/github/connector_app.go@@ -33,7 +33,7 @@ var _ Connector = (*appConnector)(nil)const githubHTTPTimeoutSeconds = 60-func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllInstallations bool) (Connector, error) {+func NewAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp, scanAllInstallations bool) (Connector, error) {var installationID int64var err error@@ -53,7 +53,7 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllInreturn nil, fmt.Errorf("could not parse app ID %q: %w", app.AppId, err)}- installationClient, err := GetAppClient(context.Background(), apiEndpoint, appID, app.PrivateKey)+ installationClient, err := GetAppClient(ctx, apiEndpoint, appID, app.PrivateKey)if err != nil {return nil, fmt.Errorf("could not create installation client: %w", err)}@@ -72,7 +72,7 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllIn// clients created lazily per-repo, so there is no default client to// validate up front.if connector.HasDefaultInstallation() {- _, err := connector.clientsForInstallation(installationID)+ _, err := connector.clientsForInstallation(ctx, installationID)if err != nil {return nil, fmt.Errorf("could not create clients for configured installation: %w", err)}@@ -98,7 +98,7 @@ func (c *appConnector) HasDefaultInstallation() bool {// as a fallback client for repos/resources outside any installation listing// and call it unconditionally.func (c *appConnector) APIClient() *github.Client {- client, err := c.APIClientForInstallation(c.installationID)+ client, err := c.APIClientForInstallation(contextForClientLookup(), c.installationID)if err != nil {return nil}@@ -107,16 +107,16 @@ func (c *appConnector) APIClient() *github.Client {func (c *appConnector) APIClientForRepo(repoURL string) (*github.Client, error) {installID, _ := c.installationIDForRepo(repoURL)- return c.APIClientForInstallation(installID)+ return c.APIClientForInstallation(contextForClientLookup(), installID)}func (c *appConnector) GraphQLClientForRepo(ctx context.Context, repoURL string) (*githubv4.Client, error) {installID, _ := c.installationIDForRepo(repoURL)- return c.graphqlClientForInstallation(installID)+ return c.graphqlClientForInstallation(ctx, installID)}-func (c *appConnector) graphqlClientForInstallation(installID int64) (*githubv4.Client, error) {- clients, err := c.clientsForInstallation(installID)+func (c *appConnector) graphqlClientForInstallation(ctx context.Context, installID int64) (*githubv4.Client, error) {+ clients, err := c.clientsForInstallation(ctx, installID)if err != nil {return nil, err}@@ -136,7 +136,7 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...stringreturn "", nil, fmt.Errorf("no GitHub App installation resolved for repo %q; set githubApp.installationId to a fallback installation to scan repos outside installation listings (e.g. member repos with scanUsers) together with scanAllInstallations", repoURL)}- cs, err := c.clientsForInstallation(installID)+ cs, err := c.clientsForInstallation(ctx, installID)if err != nil {return "", nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)}@@ -159,7 +159,7 @@ func (c *appConnector) grantsForRepo(ctx context.Context, repoURL string) (*instreturn nil, fmt.Errorf("no GitHub App installation resolved for repo %q", repoURL)}- cs, err := c.clientsForInstallation(installID)+ cs, err := c.clientsForInstallation(ctx, installID)if err != nil {return nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)}@@ -242,7 +242,7 @@ func (c *appConnector) setRepoInstallationForWiki(repoURL string, installationID// installation, lazily creating it if needed. See APIClient for why this// cannot simply return nil when no default installation is configured.func (c *appConnector) GraphQLClient() *githubv4.Client {- client, err := c.graphqlClientForInstallation(c.installationID)+ client, err := c.graphqlClientForInstallation(contextForClientLookup(), c.installationID)if err != nil {return nil}@@ -257,8 +257,8 @@ func (c *appConnector) InstallationClient() *github.Client {// installation. This is needed when a GitHub App is installed across multiple// orgs — each org's API calls must use that org's installation token to get// proper IP allowlist bypass and permission scoping.-func (c *appConnector) APIClientForInstallation(installationID int64) (*github.Client, error) {- clients, err := c.clientsForInstallation(installationID)+func (c *appConnector) APIClientForInstallation(ctx context.Context, installationID int64) (*github.Client, error) {+ clients, err := c.clientsForInstallation(ctx, installationID)if err != nil {return nil, err}@@ -266,6 +266,19 @@ func (c *appConnector) APIClientForInstallation(installationID int64) (*github.Creturn clients.APIClient, nil}-func (c *appConnector) clientsForInstallation(installationID int64) (*AppInstallationClients, error) {- return GetAppInstallationClients(context.Background(), c.apiEndpoint, c.appID, c.appPrivateKey, installationID)+func (c *appConnector) clientsForInstallation(ctx context.Context, installationID int64) (*AppInstallationClients, error) {+ return GetAppInstallationClients(ctx, c.apiEndpoint, c.appID, c.appPrivateKey, installationID)+}++// contextForClientLookup is only for the Connector methods that don't take a+// context: APIClient, APIClientForRepo and GraphQLClient. They only build a+// client or take it from the cache, which makes no network call. On a cache+// miss, though, the "Creating GraphQL client" line is logged without the+// source fields and ignores per-source debug. The fix is to pass a context+// through the Connector interface, which is left for a separate change since+// it touches every connector and its callers. Everything else passes its own+// context. That is why the semgrep rule against new contexts is turned off on+// this line.+func contextForClientLookup() context.Context {+ return context.Background() // nosemgrep}diff --git a/pkg/sources/github/connector_app_test.go b/pkg/sources/github/connector_app_test.goindex a0ac2fe6c..82e39da1f 100644--- a/pkg/sources/github/connector_app_test.go+++ b/pkg/sources/github/connector_app_test.go@@ -56,7 +56,7 @@ func TestAPIClientForInstallation(t *testing.T) {}))defer server.Close()- connector, err := NewAppConnector(+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()),server.URL,&credentialspb.GitHubApp{PrivateKey: string(privKey),@@ -69,11 +69,11 @@ func TestAPIClientForInstallation(t *testing.T) {ac := connector.(*appConnector)t.Run("creates distinct clients for different installations", func(t *testing.T) {- client1, err := ac.APIClientForInstallation(111)+ client1, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 111)require.NoError(t, err)assert.NotNil(t, client1)- client2, err := ac.APIClientForInstallation(222)+ client2, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 222)require.NoError(t, err)assert.NotNil(t, client2)@@ -81,9 +81,9 @@ func TestAPIClientForInstallation(t *testing.T) {})t.Run("reuses clients for the same installation", func(t *testing.T) {- client1, err := ac.APIClientForInstallation(333)+ client1, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 333)require.NoError(t, err)- client2, err := ac.APIClientForInstallation(333)+ client2, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 333)require.NoError(t, err)assert.Same(t, client1, client2)@@ -94,7 +94,7 @@ func TestAPIClientForInstallation(t *testing.T) {tokenRequestPaths = nilmu.Unlock()- client, err := ac.APIClientForInstallation(42)+ client, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 42)require.NoError(t, err)ctx := trContext.Background()@@ -113,11 +113,11 @@ func TestAPIClientForInstallation(t *testing.T) {ctx := trContext.Background()- client1, err := ac.APIClientForInstallation(100)+ client1, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 100)require.NoError(t, err)_, _, _ = client1.Organizations.ListMembers(ctx, "org-a", nil)- client2, err := ac.APIClientForInstallation(200)+ client2, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 200)require.NoError(t, err)_, _, _ = client2.Organizations.ListMembers(ctx, "org-b", nil)@@ -153,7 +153,7 @@ func TestNewAppConnectorDefaultAPIClientUsesConfiguredInstallation(t *testing.T)}))defer server.Close()- connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{PrivateKey: string(privKey),InstallationId: "4242",AppId: "12345",@@ -190,7 +190,7 @@ func TestNewAppConnectorInstallationIDOptionalWithScanAllInstallations(t *testindefer server.Close()// scanAllInstallations=true with no installationId configured should succeed.- connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{PrivateKey: string(privKey),AppId: "12345",}, true)@@ -198,7 +198,7 @@ func TestNewAppConnectorInstallationIDOptionalWithScanAllInstallations(t *testinrequire.NotNil(t, connector)// Without scanAllInstallations, installationId is still required.- _, err = NewAppConnector(server.URL, &credentialspb.GitHubApp{+ _, err = NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{PrivateKey: string(privKey),AppId: "12345",}, false)@@ -221,7 +221,7 @@ func TestAPIClientAndGraphQLClientNonNilWithoutDefaultInstallation(t *testing.T)}))defer server.Close()- connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{PrivateKey: string(privKey),AppId: "12345",}, true)@@ -299,7 +299,7 @@ func TestAPIClientForRepoUsesRepoInstallationMap(t *testing.T) {}))defer server.Close()- connector, err := NewAppConnector(+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()),server.URL,&credentialspb.GitHubApp{PrivateKey: string(privKey),@@ -338,7 +338,7 @@ func TestGraphQLClientForRepoCachesClients(t *testing.T) {defaultGraphQLClient := &githubv4.Client{}- connector, err := NewAppConnector(+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()),server.URL,&credentialspb.GitHubApp{PrivateKey: string(privKey),diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex c8af360a5..3e969d088 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -32,6 +32,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/feature""github.com/trufflesecurity/trufflehog/v3/pkg/giturl""github.com/trufflesecurity/trufflehog/v3/pkg/handlers"+ "github.com/trufflesecurity/trufflehog/v3/pkg/log""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"@@ -229,7 +230,8 @@ func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string]for _, ig := range include {g, err := glob.Compile(ig)if err != nil {- ctx.Logger().V(1).Info("invalid include glob", "include_value", ig, "err", err)+ // the bad glob is dropped, so tell the user this filter is not used+ ctx.Logger().Error(err, "invalid include glob; ignoring it", "include_value", ig)continue}includeGlobs = append(includeGlobs, g)@@ -237,7 +239,8 @@ func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string]for _, eg := range exclude {g, err := glob.Compile(eg)if err != nil {- ctx.Logger().V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)+ // the bad glob is dropped, so tell the user this filter is not used+ ctx.Logger().Error(err, "invalid exclude glob; ignoring it", "exclude_value", eg)continue}excludeGlobs = append(excludeGlobs, g)@@ -333,9 +336,15 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so)s.repos = s.conn.Repositoriesfor _, repo := range s.repos {+ // safeRepoURL keeps the credential out of the lines we write, this+ // also catches it in any string field logged elsewhere+ log.RedactGlobally(repoCredential(repo))r, err := s.normalizeRepo(repo)if err != nil {- aCtx.Logger().Error(err, "invalid repository", "repo", repo)+ // the repo comes from the user's config and may hold a token, so strip it,+ // also from the error, which repeats the URL+ safeErr := errors.New(strings.ReplaceAll(err.Error(), repo, safeRepoURL(repo)))+ aCtx.Logger().Error(safeErr, "invalid repository", "repo", safeRepoURL(repo))continue}s.filteredRepoCache.Set(repo, r)@@ -395,6 +404,26 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so}s.git = git.NewGit(cfg)+ // log the config choices so support can see how the source was set up+ // only counts and flags, never repo URLs, tokens or keys+ aCtx.Logger().V(1).Info("github source initialized",+ "auth_type", fmt.Sprintf("%T", s.conn.GetCredential()),+ "endpoint", safeRepoURL(s.conn.GetEndpoint()),+ "repo_count", len(s.conn.GetRepositories()),+ "org_count", len(s.conn.GetOrganizations()),+ "include_repo_count", len(s.conn.GetIncludeRepos()),+ "ignore_repo_count", len(s.conn.GetIgnoreRepos()),+ "include_forks", s.conn.GetIncludeForks(),+ "exclude_archived", s.conn.GetExcludeArchived(),+ "scan_users", s.conn.GetScanUsers(),+ "scan_all_installations", s.conn.GetScanAllInstallations(),+ "ignore_gists", s.ignoreGists,+ "include_wikis", s.conn.GetIncludeWikis(),+ "include_issue_comments", s.includeIssueComments,+ "include_pr_comments", s.includePRComments,+ "include_gist_comments", s.includeGistComments,+ "comments_timeframe_days", s.commentsTimeframeDays,+ )return nil}@@ -565,7 +594,7 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e// Final filter check - only include repositories that pass the filterif s.filteredRepoCache.wantRepo(repoName) {- ctx := context.WithValue(ctx, "repo", repo)+ ctx := context.WithValue(ctx, "repo", safeRepoURL(repo))repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})if err != nil {@@ -591,7 +620,7 @@ func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string, reporterif _, ok := s.repoInfoCache.get(repo); ok {return repo, nil}- ctx.Logger().V(2).Info("Caching repository info")+ ctx.Logger().V(2).Info("Caching repository info", "repo", safeRepoURL(repo))_, urlParts, err := getRepoURLParts(repo)if err != nil {@@ -743,7 +772,6 @@ func (s *Source) enumerateWithToken(ctx context.Context, isGithubEnterprise bool}if s.conn.ScanUsers && len(s.memberCache) > 0 {- ctx.Logger().Info("Fetching repos for org members", "org_count", s.orgsCache.Count(), "member_count", len(s.memberCache))s.addReposForMembers(ctx, reporter)}}@@ -770,7 +798,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector,if err != nil {return err}- ctx.Logger().Info("Scanning repos", "org_members", len(s.memberCache))+ ctx.Logger().V(1).Info("Scanning repos", "org_members", len(s.memberCache))if !connector.HasDefaultInstallation() {// Member repos/gists live outside any installation's own// listing, so there is no installation token to fetch them@@ -789,11 +817,18 @@ func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector,ctx.Logger().Info("Skipping org member gists: GitHub doesn't allow listing gists with a GitHub App installation token", "org_members", len(s.memberCache))}for member := range s.memberCache {- logger := ctx.Logger().WithValues("member", member)+ // once the scan is cancelled every fetch fails, so stop here+ // instead of logging an error for each remaining member+ if common.IsDone(ctx) {+ ctx.Logger().V(4).Info("skipping remaining members", "reason", "scan cancelled")+ return nil+ }+ // add the member to the error line below so we know whose fetch failed+ memberCtx := context.WithValues(ctx, "member", member)// TODO: Add authenticated user list repo for app token. It does support as per docs but need to test it before we enable it here.// docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-the-authenticated-user- if err := s.getReposByUser(ctx, member, false, reporter); err != nil {- logger.Error(err, "error fetching repos by user")+ if err := s.getReposByUser(ctx, member, false, reporter); err != nil && ctx.Err() == nil {+ memberCtx.Logger().Error(err, "error fetching repos by user")}}}@@ -890,9 +925,9 @@ func (s *Source) enumerateAllInstallationRepos(ctx context.Context, connector *ainstallID := install.GetID()account := install.GetAccount().GetLogin()installCtx := context.WithValues(ctx, "org", account, "installation_id", installID)- installCtx.Logger().Info("Enumerating repos from GitHub App installation")+ installCtx.Logger().V(2).Info("Enumerating repos from GitHub App installation")- client, err := connector.APIClientForInstallation(installID)+ client, err := connector.APIClientForInstallation(installCtx, installID)if err != nil {installCtx.Logger().Error(err, "could not create API client for installation")if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", installID, err)); err != nil {@@ -981,7 +1016,7 @@ func (s *Source) mapReposToInstallations(ctx context.Context, connector *appConninstalledRepoKeys := make(map[int64]map[string]struct{}, len(installs))for _, install := range installs {installID := install.GetID()- client, err := connector.APIClientForInstallation(installID)+ client, err := connector.APIClientForInstallation(ctx, installID)if err != nil {errs = errors.Join(errs, fmt.Errorf("could not create API client for installation %d: %w", installID, err))continue@@ -1199,8 +1234,9 @@ func (s *Source) mapRemainingAccessibleRepos(continue}+ // requested.original is the repo as the user wrote it, so strip any tokenctx.Logger().Info("repo is not owned by any app installation; scanning with the default installation token",- "repo", requested.original)+ "repo", safeRepoURL(requested.original))deleteRepoMappingRequest(wantedRepos, requested)s.recordRepoInstallation(connector, connector.installationID, repo, requested)break@@ -1314,18 +1350,25 @@ func (s *Source) repoURLFromTargetLink(link string) (string, error) {func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error {var scannedCount uint64 = 1- ctx.Logger().V(2).Info("Found repos to scan", "count", len(s.repos))+ ctx.Logger().V(1).Info("Found repos to scan", "count", len(s.repos))// If there is resume information available, limit this scan to only the repos that still need scanning.reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)+ // repos finished by an earlier, interrupted run are left out, so say how many+ if skipped := len(s.repos) - len(reposToScan); skipped > 0 {+ ctx.Logger().V(1).Info("resuming scan; skipping repos already scanned", "skipped", skipped, "remaining", len(reposToScan))+ }s.repos = reposToScanfor i, repoURL := range s.repos {s.jobPool.Go(func() error {+ // repos the user listed in the config are used as-is and may hold a+ // token, so only the cleaned URL goes into logsif common.IsDone(ctx) {+ ctx.Logger().V(4).Info("skipping repo", "repo", safeRepoURL(repoURL), "reason", "scan cancelled")return nil}- ctx := context.WithValue(ctx, "repo", repoURL)+ ctx := context.WithValue(ctx, "repo", safeRepoURL(repoURL))// TODO: set progress complete is being called concurrently with is.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)@@ -1373,23 +1416,35 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.}// Scan the wiki, if enabled, and the repo has one.- if s.conn.IncludeWikis && repoInfo.hasWiki && s.wikiIsReachable(ctx, repoURL) {+ // The checks are split up only so each skip can say why.+ switch {+ case !s.conn.IncludeWikis:+ // wikis are off for the whole source, nothing to log per repo+ case !repoInfo.hasWiki:+ ctx.Logger().V(4).Info("skipping wiki", "reason", "repo has no wiki")+ case !s.wikiIsReachable(ctx, repoURL):+ ctx.Logger().V(4).Info("skipping wiki", "reason", "wiki is not reachable")+ default:wikiURL, ok := wikiCloneURLForRepoInfo(repoURL, repoInfo)- if ok {- wikiCtx := context.WithValue(ctx, "repo", wikiURL)+ if !ok {+ ctx.Logger().V(4).Info("skipping wiki", "reason", "could not build wiki clone URL")+ break+ }+ wikiCtx := context.WithValue(ctx, "repo", safeRepoURL(wikiURL))- _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)- if err != nil {- // Ignore "Repository not found" errors.- // It's common for GitHub's API to say a repo has a wiki when it doesn't.- if !strings.Contains(err.Error(), "not found") {- if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil {- return err- }+ _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)+ if err != nil {+ // Ignore "Repository not found" errors.+ // It's common for GitHub's API to say a repo has a wiki when it doesn't.+ if strings.Contains(err.Error(), "not found") {+ wikiCtx.Logger().V(4).Info("skipping wiki", "reason", "GitHub says the wiki was not found")+ } else {+ if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil {+ return err}-- // Don't return, it still might be possible to scan comments.}++ // Don't return, it still might be possible to scan comments.}}@@ -1517,7 +1572,7 @@ func (s *Source) handleRateLimit(ctx context.Context, errIn error, reporters ...if retryAfter > 0 {retryAfter = retryAfter + jitterrateLimitResumeTime = now.Add(retryAfter)- ctx.Logger().Info(fmt.Sprintf("exceeded %s rate limit", limitType), "retry_after", retryAfter.String(), "resume_time", rateLimitResumeTime.Format(time.RFC3339))+ ctx.Logger().Info("exceeded rate limit", "limit_type", limitType, "retry_after", retryAfter.String(), "resume_time", rateLimitResumeTime.Format(time.RFC3339))// Only report the error if a reporter was providedfor _, reporter := range reporters {_ = reporter.Err(ctx, fmt.Errorf("exceeded %s rate limit", limitType))@@ -1567,13 +1622,21 @@ func (s *Source) handleRateLimitWithChunkReporter(ctx context.Context, reporter}func (s *Source) addReposForMembers(ctx context.Context, reporter sources.UnitReporter) {- ctx.Logger().Info("Fetching repos from members", "members", len(s.memberCache))+ ctx.Logger().V(1).Info("Fetching repos from members", "members", len(s.memberCache), "org_count", s.orgsCache.Count())for member := range s.memberCache {- if err := s.addUserGistsToCache(ctx, member, reporter); err != nil {- ctx.Logger().Info("Unable to fetch gists by user", "user", member, "error", err)+ // once the scan is cancelled every fetch fails, so stop here+ // instead of logging an error for each remaining member+ if common.IsDone(ctx) {+ ctx.Logger().V(4).Info("skipping remaining members", "reason", "scan cancelled")+ return+ }+ // a fetch that was running when the scan got cancelled fails too,+ // that is not an error worth logging+ if err := s.addUserGistsToCache(ctx, member, reporter); err != nil && ctx.Err() == nil {+ ctx.Logger().Error(err, "Unable to fetch gists by user", "user", member)}- if err := s.getReposByUser(ctx, member, false, reporter); err != nil {- ctx.Logger().Info("Unable to fetch repos by user", "user", member, "error", err)+ if err := s.getReposByUser(ctx, member, false, reporter); err != nil && ctx.Err() == nil {+ ctx.Logger().Error(err, "Unable to fetch repos by user", "user", member)}}}@@ -1586,7 +1649,8 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporter}gistOpts := &github.GistListOptions{}- logger := ctx.Logger().WithValues("user", user)+ // every log line below is about this user+ ctx = context.WithValues(ctx, "user", user)for {gists, res, err := s.connector.APIClient().Gists.List(ctx, user, gistOpts)@@ -1608,7 +1672,8 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporterif res == nil || res.NextPage == 0 {break}- logger.V(2).Info("Listed gists", "page", gistOpts.Page, "last_page", res.LastPage)+ // one line per page, with how many API calls we have left before the rate limit+ ctx.Logger().V(4).Info("Listed gists", "page", gistOpts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)gistOpts.Page = res.NextPage}return nil@@ -1631,7 +1696,7 @@ func (s *Source) addMembersByApp(ctx context.Context, connector *appConnector, r// Create a per-installation API client so that the token is scoped to// this org. GitHub App installation tokens only bypass IP allowlists for// their own org; using a cross-org token causes 403s.- client, err := connector.APIClientForInstallation(install.GetID())+ client, err := connector.APIClientForInstallation(installCtx, install.GetID())if err != nil {installCtx.Logger().Error(err, "could not create API client for installation")if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", install.GetID(), err)); err != nil {@@ -1678,7 +1743,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context, reporter sources.UnitRep}lastOrgID := *orgs[len(orgs)-1].ID- ctx.Logger().V(2).Info(fmt.Sprintf("listed organization IDs %d through %d", orgOpts.Since, lastOrgID))+ ctx.Logger().V(4).Info("listed organization IDs", "from_id", orgOpts.Since, "to_id", lastOrgID)orgOpts.Since = lastOrgIDfor _, org := range orgs {@@ -1701,18 +1766,20 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string, reporter sourceorgOpts := &github.ListOptions{PerPage: defaultPagination,}- logger := ctx.Logger().WithValues("user", user)+ // every log line below is about this user+ ctx = context.WithValues(ctx, "user", user)for {orgs, resp, err := s.connector.APIClient().Organizations.List(ctx, "", orgOpts)if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {continue}if err != nil {- logger.Error(err, "Could not list organizations")+ ctx.Logger().Error(err, "Could not list organizations")return}- logger.V(2).Info("Listed orgs", "page", orgOpts.Page, "last_page", resp.LastPage)+ // one line per page, with how many API calls we have left before the rate limit+ ctx.Logger().V(4).Info("Listed orgs", "page", orgOpts.Page, "last_page", resp.LastPage, "rate_remaining", resp.Rate.Remaining)for _, org := range orgs {if org.Login == nil {continue@@ -1738,7 +1805,8 @@ func (s *Source) addMembersByOrgWithClient(ctx context.Context, client *github.C},}- logger := ctx.Logger().WithValues("org", org)+ // every log line below is about this org+ ctx = context.WithValues(ctx, "org", org)for {members, res, err := client.Organizations.ListMembers(ctx, org, opts)if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {@@ -1751,7 +1819,8 @@ func (s *Source) addMembersByOrgWithClient(ctx context.Context, client *github.Creturn fmt.Errorf("organization (%q) had 0 members: account may not have access to list organization members", org)}- logger.V(2).Info("Listed members", "page", opts.Page, "last_page", res.LastPage)+ // one line per page, with how many API calls we have left before the rate limit+ ctx.Logger().V(4).Info("Listed members", "page", opts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)for _, m := range members {usr := m.Loginif usr == nil || *usr == "" {@@ -2614,7 +2683,7 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget,if meta.GetFile() == "" && meta.GetCommit() != "" {ctx := context.WithValues(ctx, "commit_hash", meta.GetCommit())- ctx.Logger().V(2).Info("secret metadata has no file; scanning commit metadata instead")+ ctx.Logger().V(4).Info("secret metadata has no file; scanning commit metadata instead")return s.scanCommitMetadata(ctx, apiClient, segments[1], segments[2], meta, &chunkSkel, reporter)}@@ -2777,7 +2846,7 @@ func (s *Source) scanCommitMetadata(ctx context.Context, apiClient *github.Clienfunc (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {repoURL, kind := unit.SourceUnitID()- ctx = context.WithValue(ctx, "repo", repoURL)+ ctx = context.WithValue(ctx, "repo", safeRepoURL(repoURL))// If the unit has an installation ID, use it. Fetching it is slow,// specifically it stalls large multi-org scans on rate limits (INT-790).diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex e71c2f289..b00ba3908 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -19,16 +19,21 @@ import ("testing""time"+ "github.com/go-logr/zapr""github.com/google/go-cmp/cmp""github.com/google/go-github/v67/github""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "go.uber.org/zap"+ "go.uber.org/zap/zapcore"+ "go.uber.org/zap/zaptest/observer""golang.org/x/sync/errgroup""google.golang.org/protobuf/types/known/anypb""gopkg.in/h2non/gock.v1""github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/log/logtest""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"@@ -377,7 +382,7 @@ func TestAppConnector_EnterpriseBaseURL(t *testing.T) {privateKey := createPrivateKey()enterpriseEndpoint := "https://api.example.ghe.com"- connector, err := NewAppConnector(enterpriseEndpoint, &credentialspb.GitHubApp{+ connector, err := NewAppConnector(context.Background(), enterpriseEndpoint, &credentialspb.GitHubApp{PrivateKey: privateKey,InstallationId: "1337",AppId: "4141",@@ -3047,3 +3052,192 @@ func TestCreateAPIClient_CloudGitHub(t *testing.T) {t.Errorf("expected https://api.github.com/, got: %s", client.BaseURL.String())}}++func TestEnumerateWithToken_DebugLogging(t *testing.T) {+ const testToken = "ghp_TESTTOKENdonotlog1234567890abcdef"++ // mock one page of repos: one to scan, plus one of each kind we skip+ mockAPI := func() {+ gock.New("https://api.github.com").+ Get("/user").+ Reply(200).+ JSON(map[string]string{"login": "test-user"})++ gock.New("https://api.github.com").+ Get("/user/repos").+ Reply(200).+ JSON([]map[string]any{+ {"full_name": "test-user/kept-repo", "clone_url": "https://github.com/test-user/kept-repo.git"},+ {"full_name": "test-user/forked-repo", "clone_url": "https://github.com/test-user/forked-repo.git", "fork": true},+ {"full_name": "test-user/archived-repo", "clone_url": "https://github.com/test-user/archived-repo.git", "archived": true},+ {"full_name": "test-user/ignored-repo", "clone_url": "https://github.com/test-user/ignored-repo.git"},+ })++ gock.New("https://api.github.com").+ Get("/user/orgs").+ Reply(200).+ JSON(`[]`)++ gock.New("https://api.github.com").+ Get("/users/test-user/gists").+ Reply(200).+ JSON(`[]`)+ }++ // init the source with the given context and list the repos+ runEnumerate := func(t *testing.T, ctx context.Context) *Source {+ s, conn := createTestSource(&sourcespb.GitHub{+ Endpoint: "https://api.github.com",+ Credential: &sourcespb.GitHub_Token{Token: testToken},+ ExcludeArchived: true,+ IgnoreRepos: []string{"test-user/ignored-repo"},+ })+ assert.NoError(t, s.Init(ctx, "test - github", 1, 1337, false, conn, 1))+ gock.InterceptClient(s.connector.APIClient().Client())++ assert.NoError(t, s.enumerateWithToken(ctx, false, noopReporter()))+ assert.True(t, gock.IsDone())+ return s+ }++ t.Run("debug on", func(t *testing.T) {+ defer gock.Off()+ mockAPI()++ ctx, logs := logtest.DebugContext()+ ctx = context.WithValues(ctx, "source_name", "test - github", "job_id", 1)+ s := runEnumerate(t, ctx)+ assert.True(t, s.filteredRepoCache.Exists("test-user/kept-repo"))++ // the config line from Init shows the auth type and filters, not the token+ initLines := logs.FilterMessage("github source initialized").All()+ if assert.Len(t, initLines, 1) {+ fields := initLines[0].ContextMap()+ assert.Equal(t, "*sourcespb.GitHub_Token", fields["auth_type"])+ assert.Equal(t, int64(1), fields["ignore_repo_count"])+ assert.Equal(t, true, fields["exclude_archived"])+ }++ // the page line and one skip line for each skipped repo, with the reason+ assert.Equal(t, 1, logs.FilterMessage("Listed repos").Len())+ reasons := map[string]string{}+ for _, entry := range logs.FilterMessage("skipping repository").All() {+ fields := entry.ContextMap()+ reasons[fields["repo"].(string)] = fields["reason"].(string)+ }+ assert.Equal(t, map[string]string{+ "test-user/forked-repo": "forks are not included",+ "test-user/archived-repo": "archived repos are excluded",+ "test-user/ignored-repo": "excluded by include or ignore filter",+ }, reasons)++ // every line came from the context logger, so it has the source fields,+ // and no line contains the token+ for _, entry := range logs.All() {+ fields := entry.ContextMap()+ assert.Contains(t, fields, "source_name", "line %q", entry.Message)+ assert.Contains(t, fields, "job_id", "line %q", entry.Message)+ assert.NotContains(t, entry.Message, testToken)+ assert.NotContains(t, fmt.Sprint(fields), testToken)+ }+ })++ t.Run("debug off", func(t *testing.T) {+ defer gock.Off()+ mockAPI()++ core, logs := observer.New(zapcore.InfoLevel)+ ctx := context.WithLogger(context.Background(), zapr.NewLogger(zap.New(core)))+ runEnumerate(t, ctx)++ assert.Zero(t, logs.FilterMessage("Listed repos").Len())+ assert.Zero(t, logs.FilterMessage("skipping repository").Len())+ })+}++func TestScan_LogsResumeAndCancelSkips(t *testing.T) {+ const testToken = "ghp_TESTTOKENinrepourl1234567890"++ s := initTestSource(&sourcespb.GitHub{+ Endpoint: "https://api.github.com",+ Credential: &sourcespb.GitHub_Token{Token: "token"},+ })+ s.repos = []string{+ "https://github.com/org/a.git",+ "https://github.com/org/b.git",+ // a repo from the config with a token in it, which must never be logged+ "https://user:" + testToken + "@github.com/org/c.git",+ }+ // pretend an earlier run stopped while scanning b, so a is already done+ s.SetProgressComplete(0, 3, "", sources.EncodeResumeInfo([]string{"https://github.com/org/b.git"}))++ // cancel before the scan starts, so no repo is cloned and each one is+ // skipped with a log line instead+ debugCtx, logs := logtest.DebugContext()+ ctx, cancel := context.WithCancel(debugCtx)+ cancel()++ assert.NoError(t, s.scan(ctx, &collectingReporter{}))++ resumeLines := logs.FilterMessage("resuming scan; skipping repos already scanned").All()+ if assert.Len(t, resumeLines, 1) {+ assert.Equal(t, int64(1), resumeLines[0].ContextMap()["skipped"])+ assert.Equal(t, int64(2), resumeLines[0].ContextMap()["remaining"])+ }++ var cancelled []string+ for _, entry := range logs.FilterMessage("skipping repo").All() {+ assert.Equal(t, "scan cancelled", entry.ContextMap()["reason"])+ cancelled = append(cancelled, entry.ContextMap()["repo"].(string))+ }+ assert.ElementsMatch(t, []string{"https://github.com/org/b.git", "https://github.com/org/c.git"}, cancelled)++ for _, entry := range logs.All() {+ assert.NotContains(t, entry.Message, testToken)+ assert.NotContains(t, fmt.Sprint(entry.ContextMap()), testToken)+ }+}++func TestInit_InvalidRepoErrorDoesNotLogToken(t *testing.T) {+ const testToken = "ghp_TESTTOKENinvalidrepo1234567890"++ // a repo from the config with a token in it and no repo name, so it fails+ // to normalize and the error repeats the URL+ s, conn := createTestSource(&sourcespb.GitHub{+ Endpoint: "https://api.github.com",+ Credential: &sourcespb.GitHub_Token{Token: "token"},+ Repositories: []string{"https://" + testToken + "@github.com/org"},+ })++ ctx, logs := logtest.DebugContext()+ assert.NoError(t, s.Init(ctx, "test - github", 0, 1337, false, conn, 1))++ errLines := logs.FilterMessage("invalid repository").All()+ if assert.Len(t, errLines, 1) {+ fields := errLines[0].ContextMap()+ assert.Equal(t, "https://github.com/org", fields["repo"])+ assert.Contains(t, fields["error"], "missing the repo name")+ }+ for _, entry := range logs.All() {+ assert.NotContains(t, entry.Message, testToken)+ assert.NotContains(t, fmt.Sprint(entry.ContextMap()), testToken)+ }+}++func TestAddReposForMembers_StopsOnCancel(t *testing.T) {+ s := initTestSource(&sourcespb.GitHub{+ Endpoint: "https://api.github.com",+ Credential: &sourcespb.GitHub_Token{Token: "token"},+ })+ s.memberCache = map[string]struct{}{"member1": {}, "member2": {}}++ // cancel before the loop starts, so no member is fetched and no error is logged+ debugCtx, logs := logtest.DebugContext()+ ctx, cancel := context.WithCancel(debugCtx)+ cancel()++ s.addReposForMembers(ctx, noopReporter())++ assert.Equal(t, 1, logs.FilterMessage("skipping remaining members").Len())+ assert.Zero(t, logs.FilterLevelExact(zapcore.ErrorLevel).Len())+}diff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.goindex 4efb21c17..6af4c1cad 100644--- a/pkg/sources/github/graphql.go+++ b/pkg/sources/github/graphql.go@@ -45,7 +45,7 @@ func (s *Source) processIssuesWithComments(totalIssues += len(query.GetIssues())- ctx.Logger().V(5).Info("Scanning Issues",+ ctx.Logger().V(4).Info("Scanning Issues","total_issues", len(query.GetIssues()))if err := s.chunkGraphqlIssues(ctx, repoInfo, query.Repository.Issues.Nodes, reporter); err != nil {@@ -54,7 +54,7 @@ func (s *Source) processIssuesWithComments(// process each issue commentsfor _, issue := range query.GetIssues() {- ctx.Logger().V(5).Info("Scanning Issue Comments",+ ctx.Logger().V(3).Info("Scanning Issue Comments","issue_id", issue.Number,"total_comments", len(issue.GetIssueComments()),)@@ -84,7 +84,7 @@ func (s *Source) processIssuesWithComments(return fmt.Errorf("error fetching issue: %w", err)}- ctx.Logger().V(5).Info("Scanning additional issue comments",+ ctx.Logger().V(4).Info("Scanning additional issue comments","issue_id", issue.Number,"total_comments", len(commentsQuery.GetIssueComments()))@@ -137,7 +137,7 @@ func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githutotalPRs += len(query.GetPullRequests())- ctx.Logger().V(5).Info("Scanning pull requests",+ ctx.Logger().V(4).Info("Scanning pull requests","total_pull_requests", len(query.GetPullRequests()))if err := s.chunkGraphqlPullRequests(ctx, repoInfo, query.GetPullRequests(), reporter); err != nil {@@ -146,7 +146,7 @@ func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githu// process each pr commentsfor _, pr := range query.GetPullRequests() {- ctx.Logger().V(5).Info("Scanning pull request comments",+ ctx.Logger().V(3).Info("Scanning pull request comments","pull_request_no", pr.Number,"total_comments", len(pr.Comments.Nodes))@@ -175,7 +175,7 @@ func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githureturn fmt.Errorf("error fetching pull request with comments: %w", err)}- ctx.Logger().V(5).Info("Scanning additional comments",+ ctx.Logger().V(4).Info("Scanning additional comments","pull_request_no", pr.Number,"total_comments", len(commentQuery.GetPRComments()))@@ -247,7 +247,7 @@ func (s *Source) processReviewThreads(ctx context.Context, graphqlClient *githubif len(threadIDs) > 0 {// process in batches of max 100for _, batch := range chunkIDs(threadIDs, 100) {- ctx.Logger().V(5).Info("Processing Thread comments in Batches", "batch_length", len(batch))+ ctx.Logger().V(4).Info("Processing Thread comments in Batches", "batch_length", len(batch))// fetch comments for the batch of threadsif err := s.fetchThreadComments(ctx, graphqlClient, batch, repoInfo, reporter, cutoffTime); err != nil {return fmt.Errorf("error fetching thread review comments: %w", err)diff --git a/pkg/sources/github/pagination.go b/pkg/sources/github/pagination.goindex 998296020..8ab23a656 100644--- a/pkg/sources/github/pagination.go+++ b/pkg/sources/github/pagination.go@@ -251,7 +251,7 @@ func walkUpdatedSince[T any](ctx context.Context, p pager, w sinceWalk[T]) (int,}windowStart = latest- ctx.Logger().V(2).Info("listing capped, continuing from last update time",+ ctx.Logger().V(4).Info("listing capped, continuing from last update time","listing", w.name, "since", windowStart.UTC().Format(time.RFC3339), "items_so_far", total)}}@@ -305,8 +305,8 @@ func fetchWithRetry[T any](ctx context.Context, p pager, fetch func() ([]T, *git}delay := transientBackoff(transientRetries)transientRetries++- ctx.Logger().V(2).Info("retrying GitHub API request after transient error",- "listing", p.listing, "retry", transientRetries, "max_retries", maxTransientRetries, "retry_in", delay.String(), "error", err.Error())+ ctx.Logger().V(4).Info("retrying GitHub API request after transient error",+ "listing", p.listing, "retry", transientRetries, "max_retries", maxTransientRetries, "retry_in", delay.String(), "reason", err.Error())if !p.wait(ctx, delay) {return items, resp, err}diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.goindex 5c62f2ede..e77262b82 100644--- a/pkg/sources/github/repo.go+++ b/pkg/sources/github/repo.go@@ -257,7 +257,8 @@ func isGitHubListingDisabled(err error) bool {// processRepos processes repositories from a source, handling pagination and rate limits.func (s *Source) processRepos(ctx context.Context, target string, reporter sources.UnitReporter, listRepos repoLister, listOpts repoListOptions) error {- logger := ctx.Logger().WithValues("target", target)+ // every log line below is about this org or user+ ctx = context.WithValues(ctx, "target", target)opts := listOpts.getListOptions()var (@@ -276,17 +277,19 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourcreturn err}- ctx.Logger().V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)+ // one line per page, with how many API calls we have left before the rate limit+ ctx.Logger().V(4).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)for _, r := range someRepos {isFork, isArchived := r.GetFork(), r.GetArchived()if isFork && !s.conn.IncludeForks {+ ctx.Logger().V(4).Info("skipping repository", "repo", r.GetFullName(), "reason", "forks are not included")continue}if isArchived && s.conn.ExcludeArchived {numArchivedSkipped++- logger.V(3).Info("skipping archived repository", "repo", r.GetFullName())+ ctx.Logger().V(4).Info("skipping repository", "repo", r.GetFullName(), "reason", "archived repos are excluded")continue}@@ -309,6 +312,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc// Check if we should process this repository based on the filterif !s.filteredRepoCache.wantRepo(repoName) {+ ctx.Logger().V(4).Info("skipping repository", "repo", repoName, "reason", "excluded by include or ignore filter")continue}@@ -334,7 +338,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourcreturn err}- logger.V(3).Info("repo attributes", "name", repoName, "kb_size", r.GetSize(), "repo_url", repoURL)+ ctx.Logger().V(2).Info("repo attributes", "name", repoName, "kb_size", r.GetSize(), "repo_url", repoURL)}if res.NextPage == 0 {@@ -344,7 +348,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc}// final logging of repository stats.- logger.V(2).Info("found repos", "total", numRepos, "num_forks", numForks, "num_archived", numArchived, "num_archived_skipped", numArchivedSkipped, "num_orgs", len(uniqueOrgs))+ ctx.Logger().V(2).Info("found repos", "total", numRepos, "num_forks", numForks, "num_archived", numArchived, "num_archived_skipped", numArchivedSkipped, "num_orgs", len(uniqueOrgs))githubOrgsEnumerated.WithLabelValues(s.name).Add(float64(len(uniqueOrgs)))return nildiff --git a/pkg/sources/github/urls.go b/pkg/sources/github/urls.goindex 7b8ee7029..ead3d101a 100644--- a/pkg/sources/github/urls.go+++ b/pkg/sources/github/urls.go@@ -6,6 +6,79 @@ import ("strings")+// safeRepoURL removes any username or password from a repo URL so it can be+// logged.+func safeRepoURL(repo string) string {+ // a normal URL like "https://user:pass@host/org/repo": let url.Parse find+ // the user info and drop it+ u, err := url.Parse(repo)+ if err == nil && u.Host != "" {+ u.User = nil+ return u.String()+ }++ // anything else is a repo name like "org/repo", or a URL that url.Parse+ // rejects, so drop the user info by hand+ if prefix, _, rest, found := splitUserInfo(repo); found {+ return prefix + rest+ }+ return repo+}++// splitUserInfo splits a repo that url.Parse rejects, like an SCP-style URL+// "user:pass@host:org/repo" or a mix like "https://token@host:org/repo", into+// the scheme prefix ("https://" or ""), the user info and the rest. The user+// info always sits before an "@" in the part before the first "/" after any+// scheme. found is false when there is no such "@".+func splitUserInfo(repo string) (prefix, userInfo, rest string, found bool) {+ scheme, afterScheme, hasScheme := strings.Cut(repo, "://")+ if hasScheme {+ prefix = scheme + "://"+ } else {+ afterScheme = repo+ }+ hostPart, _, _ := strings.Cut(afterScheme, "/")+ at := strings.LastIndex(hostPart, "@")+ if at < 0 {+ return prefix, "", afterScheme, false+ }+ return prefix, afterScheme[:at], afterScheme[at+1:], true+}++// repoCredential returns the secret in a repo URL's user info, or "" if it has+// none, so it can be redacted from logs. That is the password, or for an+// http(s) URL with no password the username, since "https://TOKEN@host" puts+// the token there. Other usernames are not secret, and redacting one like+// "git" would hide that word in every log line.+func repoCredential(repo string) string {+ var prefix, username, password string+ if u, err := url.Parse(repo); err == nil && u.Host != "" {+ if u.User == nil {+ return ""+ }+ prefix = u.Scheme + "://"+ username = u.User.Username()+ password, _ = u.User.Password()+ } else {+ var userInfo string+ var found bool+ prefix, userInfo, _, found = splitUserInfo(repo)+ if !found {+ return ""+ }+ username, password, _ = strings.Cut(userInfo, ":")+ }++ // a password can be present but empty, as in "https://TOKEN:@host"+ if password != "" {+ return password+ }+ if prefix == "http://" || prefix == "https://" {+ return username+ }+ return ""+}+func wikiCloneURLForRepo(repoURL string) (string, bool) {if !strings.HasSuffix(repoURL, ".git") || strings.HasSuffix(repoURL, ".wiki.git") {return "", falsediff --git a/pkg/sources/github/urls_test.go b/pkg/sources/github/urls_test.goindex 85bc50ee1..c1b8c82d5 100644--- a/pkg/sources/github/urls_test.go+++ b/pkg/sources/github/urls_test.go@@ -171,3 +171,42 @@ func TestIsWikiLink(t *testing.T) {assert.False(t, isWikiLink("https://github.com/owner/repo/blob/main/path/to/file"))assert.False(t, isWikiLink("not a url"))}++func TestSafeRepoURL(t *testing.T) {+ tests := map[string]string{+ "https://user:[email protected]/org/repo.git": "https://github.com/org/repo.git",+ "https://[email protected]/org/repo.git": "https://github.com/org/repo.git",+ "https://github.com/org/repo.git": "https://github.com/org/repo.git",+ "org/repo": "org/repo",+ "[email protected]:org/repo.git": "github.com:org/repo.git",+ "user:[email protected]:org/repo.git": "github.com:org/repo.git",+ // mixed HTTPS and SCP style, which url.Parse rejects+ "https://[email protected]:org/repo.git": "https://github.com:org/repo.git",+ "https://user:[email protected]:org/repo.git": "https://github.com:org/repo.git",+ }+ for input, want := range tests {+ assert.Equal(t, want, safeRepoURL(input), input)+ }+}++func TestRepoCredential(t *testing.T) {+ tests := map[string]string{+ "https://user:[email protected]/org/repo.git": "ghp_secret",+ "https://[email protected]/org/repo.git": "ghp_secret",+ "ssh://git:[email protected]/org/repo.git": "ghp_secret",+ // a plain SSH user is not a secret+ "ssh://[email protected]/org/repo.git": "",+ "https://github.com/org/repo.git": "",+ "org/repo": "",+ "[email protected]:org/repo.git": "",+ // the password is present but empty, so the username is the token+ "https://ghp_secret:@github.com/org/repo.git": "ghp_secret",+ // SCP style and mixed URLs, which url.Parse rejects+ "user:[email protected]:org/repo.git": "ghp_secret",+ "https://[email protected]:org/repo.git": "ghp_secret",+ "https://user:[email protected]:org/repo.git": "ghp_secret",+ }+ for input, want := range tests {+ assert.Equal(t, want, repoCredential(input), input)+ }+}