Skip to content

Find, verify, and analyze leaked credentials

Clean up GitHub source logging for per-source debug (#5457)

Kashif Khan0301dc36bd67
diff --git a/pkg/engine/scan.go b/pkg/engine/scan.go
index e9ed7f3c3..85e1e0184 100644
--- a/pkg/engine/scan.go
+++ b/pkg/engine/scan.go
@@ -16,12 +16,21 @@ func (e *Engine) ScanConfig(ctx context.Context, configuredSources ...sources.Co
for _, configuredSource := range configuredSources {
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, configuredSource.Name, configuredSource.SourceType())
+ // Add the source fields here, before Init, so lines logged during Init
+ // say which source and job they came from too. The source manager adds
+ // the same fields later for the scan, but skips any already set here.
+ sourceCtx := context.WithValues(ctx,
+ "source_name", configuredSource.Name,
+ "source_type", configuredSource.SourceType().String(),
+ "source_id", sourceID,
+ "job_id", jobID,
+ )
+
// A source with debug on gets a context that writes all of its debug
// logs. Both Init and the scan use it, so every log from this source is
// covered. Other sources keep the global log level.
- sourceCtx := ctx
if configuredSource.Debug {
- sourceCtx = context.WithLogLevel(ctx, log.DebugLevel)
+ sourceCtx = context.WithLogLevel(sourceCtx, log.DebugLevel)
}
source, err := configuredSource.Init(sourceCtx, sourceID, jobID)
diff --git a/pkg/engine/scan_test.go b/pkg/engine/scan_test.go
index cc9ec2094..f21d2f405 100644
--- a/pkg/engine/scan_test.go
+++ b/pkg/engine/scan_test.go
@@ -82,4 +82,21 @@ func TestScanConfig_DebugOnlyForOneSource(t *testing.T) {
assert.Equal(t, 1, countLines("chunks debug", "debug"))
assert.Zero(t, countLines("init debug", "normal"))
assert.Zero(t, countLines("chunks debug", "normal"))
+
+ // Lines from Init and from the scan both carry the source fields, and
+ // each field shows up once even though the source manager adds them too.
+ for _, message := range []string{"init info", "chunks info"} {
+ for _, entry := range logs.FilterMessage(message).FilterField(zap.String("source", "debug")).All() {
+ for _, key := range []string{"source_name", "source_type", "source_id", "job_id"} {
+ count := 0
+ for _, field := range entry.Context {
+ if field.Key == key {
+ count++
+ }
+ }
+ assert.Equal(t, 1, count, "%s on %q", key, message)
+ }
+ assert.Equal(t, "debug", entry.ContextMap()["source_name"], message)
+ }
+ }
}
diff --git a/pkg/sources/github/comments_test.go b/pkg/sources/github/comments_test.go
index c7f1ff7af..741e4e286 100644
--- a/pkg/sources/github/comments_test.go
+++ b/pkg/sources/github/comments_test.go
@@ -630,8 +630,9 @@ func TestCommentScan_RetryLogNamesTheListing(t *testing.T) {
includePRComments: true,
commentRetryWait: func(context.Context, time.Duration) bool { return true },
}
+ // retries are logged at V(4), with other API decisions
var logs bytes.Buffer
- logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(2)))
+ logger, _ := log.New("test", log.WithJSONSink(&logs, log.WithLevel(4)))
ctx := context.WithLogger(context.Background(), logger)
err := s.processIssueandPRsWithCommentsREST(ctx, gh.client, testRepo, nil, &collectingReporter{}, nil)
diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go
index 9e99e4e1f..bd4932511 100644
--- a/pkg/sources/github/connector.go
+++ b/pkg/sources/github/connector.go
@@ -117,7 +117,7 @@ func newConnector(ctx context.Context, source *Source) (Connector, error) {
switch cred := source.conn.GetCredential().(type) {
case *sourcespb.GitHub_GithubApp:
log.RedactGlobally(cred.GithubApp.GetPrivateKey())
- return NewAppConnector(apiEndpoint, cred.GithubApp, source.conn.GetScanAllInstallations())
+ return NewAppConnector(ctx, apiEndpoint, cred.GithubApp, source.conn.GetScanAllInstallations())
case *sourcespb.GitHub_BasicAuth:
log.RedactGlobally(cred.BasicAuth.GetPassword())
return NewBasicAuthConnector(ctx, apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth)
diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go
index 17e0ec0ef..60cf8753e 100644
--- a/pkg/sources/github/connector_app.go
+++ b/pkg/sources/github/connector_app.go
@@ -33,7 +33,7 @@ var _ Connector = (*appConnector)(nil)
const githubHTTPTimeoutSeconds = 60
-func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllInstallations bool) (Connector, error) {
+func NewAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp, scanAllInstallations bool) (Connector, error) {
var installationID int64
var err error
@@ -53,7 +53,7 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllIn
return nil, fmt.Errorf("could not parse app ID %q: %w", app.AppId, err)
}
- installationClient, err := GetAppClient(context.Background(), apiEndpoint, appID, app.PrivateKey)
+ installationClient, err := GetAppClient(ctx, apiEndpoint, appID, app.PrivateKey)
if err != nil {
return nil, fmt.Errorf("could not create installation client: %w", err)
}
@@ -72,7 +72,7 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllIn
// clients created lazily per-repo, so there is no default client to
// validate up front.
if connector.HasDefaultInstallation() {
- _, err := connector.clientsForInstallation(installationID)
+ _, err := connector.clientsForInstallation(ctx, installationID)
if err != nil {
return nil, fmt.Errorf("could not create clients for configured installation: %w", err)
}
@@ -98,7 +98,7 @@ func (c *appConnector) HasDefaultInstallation() bool {
// as a fallback client for repos/resources outside any installation listing
// and call it unconditionally.
func (c *appConnector) APIClient() *github.Client {
- client, err := c.APIClientForInstallation(c.installationID)
+ client, err := c.APIClientForInstallation(contextForClientLookup(), c.installationID)
if err != nil {
return nil
}
@@ -107,16 +107,16 @@ func (c *appConnector) APIClient() *github.Client {
func (c *appConnector) APIClientForRepo(repoURL string) (*github.Client, error) {
installID, _ := c.installationIDForRepo(repoURL)
- return c.APIClientForInstallation(installID)
+ return c.APIClientForInstallation(contextForClientLookup(), installID)
}
func (c *appConnector) GraphQLClientForRepo(ctx context.Context, repoURL string) (*githubv4.Client, error) {
installID, _ := c.installationIDForRepo(repoURL)
- return c.graphqlClientForInstallation(installID)
+ return c.graphqlClientForInstallation(ctx, installID)
}
-func (c *appConnector) graphqlClientForInstallation(installID int64) (*githubv4.Client, error) {
- clients, err := c.clientsForInstallation(installID)
+func (c *appConnector) graphqlClientForInstallation(ctx context.Context, installID int64) (*githubv4.Client, error) {
+ clients, err := c.clientsForInstallation(ctx, installID)
if err != nil {
return nil, err
}
@@ -136,7 +136,7 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string
return "", nil, fmt.Errorf("no GitHub App installation resolved for repo %q; set githubApp.installationId to a fallback installation to scan repos outside installation listings (e.g. member repos with scanUsers) together with scanAllInstallations", repoURL)
}
- cs, err := c.clientsForInstallation(installID)
+ cs, err := c.clientsForInstallation(ctx, installID)
if err != nil {
return "", nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)
}
@@ -159,7 +159,7 @@ func (c *appConnector) grantsForRepo(ctx context.Context, repoURL string) (*inst
return nil, fmt.Errorf("no GitHub App installation resolved for repo %q", repoURL)
}
- cs, err := c.clientsForInstallation(installID)
+ cs, err := c.clientsForInstallation(ctx, installID)
if err != nil {
return nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)
}
@@ -242,7 +242,7 @@ func (c *appConnector) setRepoInstallationForWiki(repoURL string, installationID
// installation, lazily creating it if needed. See APIClient for why this
// cannot simply return nil when no default installation is configured.
func (c *appConnector) GraphQLClient() *githubv4.Client {
- client, err := c.graphqlClientForInstallation(c.installationID)
+ client, err := c.graphqlClientForInstallation(contextForClientLookup(), c.installationID)
if err != nil {
return nil
}
@@ -257,8 +257,8 @@ func (c *appConnector) InstallationClient() *github.Client {
// installation. This is needed when a GitHub App is installed across multiple
// orgs — each org's API calls must use that org's installation token to get
// proper IP allowlist bypass and permission scoping.
-func (c *appConnector) APIClientForInstallation(installationID int64) (*github.Client, error) {
- clients, err := c.clientsForInstallation(installationID)
+func (c *appConnector) APIClientForInstallation(ctx context.Context, installationID int64) (*github.Client, error) {
+ clients, err := c.clientsForInstallation(ctx, installationID)
if err != nil {
return nil, err
}
@@ -266,6 +266,19 @@ func (c *appConnector) APIClientForInstallation(installationID int64) (*github.C
return clients.APIClient, nil
}
-func (c *appConnector) clientsForInstallation(installationID int64) (*AppInstallationClients, error) {
- return GetAppInstallationClients(context.Background(), c.apiEndpoint, c.appID, c.appPrivateKey, installationID)
+func (c *appConnector) clientsForInstallation(ctx context.Context, installationID int64) (*AppInstallationClients, error) {
+ return GetAppInstallationClients(ctx, c.apiEndpoint, c.appID, c.appPrivateKey, installationID)
+}
+
+// contextForClientLookup is only for the Connector methods that don't take a
+// context: APIClient, APIClientForRepo and GraphQLClient. They only build a
+// client or take it from the cache, which makes no network call. On a cache
+// miss, though, the "Creating GraphQL client" line is logged without the
+// source fields and ignores per-source debug. The fix is to pass a context
+// through the Connector interface, which is left for a separate change since
+// it touches every connector and its callers. Everything else passes its own
+// context. That is why the semgrep rule against new contexts is turned off on
+// this line.
+func contextForClientLookup() context.Context {
+ return context.Background() // nosemgrep
}
diff --git a/pkg/sources/github/connector_app_test.go b/pkg/sources/github/connector_app_test.go
index a0ac2fe6c..82e39da1f 100644
--- a/pkg/sources/github/connector_app_test.go
+++ b/pkg/sources/github/connector_app_test.go
@@ -56,7 +56,7 @@ func TestAPIClientForInstallation(t *testing.T) {
}))
defer server.Close()
- connector, err := NewAppConnector(
+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()),
server.URL,
&credentialspb.GitHubApp{
PrivateKey: string(privKey),
@@ -69,11 +69,11 @@ func TestAPIClientForInstallation(t *testing.T) {
ac := connector.(*appConnector)
t.Run("creates distinct clients for different installations", func(t *testing.T) {
- client1, err := ac.APIClientForInstallation(111)
+ client1, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 111)
require.NoError(t, err)
assert.NotNil(t, client1)
- client2, err := ac.APIClientForInstallation(222)
+ client2, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 222)
require.NoError(t, err)
assert.NotNil(t, client2)
@@ -81,9 +81,9 @@ func TestAPIClientForInstallation(t *testing.T) {
})
t.Run("reuses clients for the same installation", func(t *testing.T) {
- client1, err := ac.APIClientForInstallation(333)
+ client1, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 333)
require.NoError(t, err)
- client2, err := ac.APIClientForInstallation(333)
+ client2, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 333)
require.NoError(t, err)
assert.Same(t, client1, client2)
@@ -94,7 +94,7 @@ func TestAPIClientForInstallation(t *testing.T) {
tokenRequestPaths = nil
mu.Unlock()
- client, err := ac.APIClientForInstallation(42)
+ client, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 42)
require.NoError(t, err)
ctx := trContext.Background()
@@ -113,11 +113,11 @@ func TestAPIClientForInstallation(t *testing.T) {
ctx := trContext.Background()
- client1, err := ac.APIClientForInstallation(100)
+ client1, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 100)
require.NoError(t, err)
_, _, _ = client1.Organizations.ListMembers(ctx, "org-a", nil)
- client2, err := ac.APIClientForInstallation(200)
+ client2, err := ac.APIClientForInstallation(trContext.AddLogger(t.Context()), 200)
require.NoError(t, err)
_, _, _ = client2.Organizations.ListMembers(ctx, "org-b", nil)
@@ -153,7 +153,7 @@ func TestNewAppConnectorDefaultAPIClientUsesConfiguredInstallation(t *testing.T)
}))
defer server.Close()
- connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{
+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{
PrivateKey: string(privKey),
InstallationId: "4242",
AppId: "12345",
@@ -190,7 +190,7 @@ func TestNewAppConnectorInstallationIDOptionalWithScanAllInstallations(t *testin
defer server.Close()
// scanAllInstallations=true with no installationId configured should succeed.
- connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{
+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{
PrivateKey: string(privKey),
AppId: "12345",
}, true)
@@ -198,7 +198,7 @@ func TestNewAppConnectorInstallationIDOptionalWithScanAllInstallations(t *testin
require.NotNil(t, connector)
// Without scanAllInstallations, installationId is still required.
- _, err = NewAppConnector(server.URL, &credentialspb.GitHubApp{
+ _, err = NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{
PrivateKey: string(privKey),
AppId: "12345",
}, false)
@@ -221,7 +221,7 @@ func TestAPIClientAndGraphQLClientNonNilWithoutDefaultInstallation(t *testing.T)
}))
defer server.Close()
- connector, err := NewAppConnector(server.URL, &credentialspb.GitHubApp{
+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()), server.URL, &credentialspb.GitHubApp{
PrivateKey: string(privKey),
AppId: "12345",
}, true)
@@ -299,7 +299,7 @@ func TestAPIClientForRepoUsesRepoInstallationMap(t *testing.T) {
}))
defer server.Close()
- connector, err := NewAppConnector(
+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()),
server.URL,
&credentialspb.GitHubApp{
PrivateKey: string(privKey),
@@ -338,7 +338,7 @@ func TestGraphQLClientForRepoCachesClients(t *testing.T) {
defaultGraphQLClient := &githubv4.Client{}
- connector, err := NewAppConnector(
+ connector, err := NewAppConnector(trContext.AddLogger(t.Context()),
server.URL,
&credentialspb.GitHubApp{
PrivateKey: string(privKey),
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index c8af360a5..3e969d088 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -32,6 +32,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
@@ -229,7 +230,8 @@ func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string]
for _, ig := range include {
g, err := glob.Compile(ig)
if err != nil {
- ctx.Logger().V(1).Info("invalid include glob", "include_value", ig, "err", err)
+ // the bad glob is dropped, so tell the user this filter is not used
+ ctx.Logger().Error(err, "invalid include glob; ignoring it", "include_value", ig)
continue
}
includeGlobs = append(includeGlobs, g)
@@ -237,7 +239,8 @@ func (s *Source) newFilteredRepoCache(ctx context.Context, c cache.Cache[string]
for _, eg := range exclude {
g, err := glob.Compile(eg)
if err != nil {
- ctx.Logger().V(1).Info("invalid exclude glob", "exclude_value", eg, "err", err)
+ // the bad glob is dropped, so tell the user this filter is not used
+ ctx.Logger().Error(err, "invalid exclude glob; ignoring it", "exclude_value", eg)
continue
}
excludeGlobs = append(excludeGlobs, g)
@@ -333,9 +336,15 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
)
s.repos = s.conn.Repositories
for _, repo := range s.repos {
+ // safeRepoURL keeps the credential out of the lines we write, this
+ // also catches it in any string field logged elsewhere
+ log.RedactGlobally(repoCredential(repo))
r, err := s.normalizeRepo(repo)
if err != nil {
- aCtx.Logger().Error(err, "invalid repository", "repo", repo)
+ // the repo comes from the user's config and may hold a token, so strip it,
+ // also from the error, which repeats the URL
+ safeErr := errors.New(strings.ReplaceAll(err.Error(), repo, safeRepoURL(repo)))
+ aCtx.Logger().Error(safeErr, "invalid repository", "repo", safeRepoURL(repo))
continue
}
s.filteredRepoCache.Set(repo, r)
@@ -395,6 +404,26 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
}
s.git = git.NewGit(cfg)
+ // log the config choices so support can see how the source was set up
+ // only counts and flags, never repo URLs, tokens or keys
+ aCtx.Logger().V(1).Info("github source initialized",
+ "auth_type", fmt.Sprintf("%T", s.conn.GetCredential()),
+ "endpoint", safeRepoURL(s.conn.GetEndpoint()),
+ "repo_count", len(s.conn.GetRepositories()),
+ "org_count", len(s.conn.GetOrganizations()),
+ "include_repo_count", len(s.conn.GetIncludeRepos()),
+ "ignore_repo_count", len(s.conn.GetIgnoreRepos()),
+ "include_forks", s.conn.GetIncludeForks(),
+ "exclude_archived", s.conn.GetExcludeArchived(),
+ "scan_users", s.conn.GetScanUsers(),
+ "scan_all_installations", s.conn.GetScanAllInstallations(),
+ "ignore_gists", s.ignoreGists,
+ "include_wikis", s.conn.GetIncludeWikis(),
+ "include_issue_comments", s.includeIssueComments,
+ "include_pr_comments", s.includePRComments,
+ "include_gist_comments", s.includeGistComments,
+ "comments_timeframe_days", s.commentsTimeframeDays,
+ )
return nil
}
@@ -565,7 +594,7 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
// Final filter check - only include repositories that pass the filter
if s.filteredRepoCache.wantRepo(repoName) {
- ctx := context.WithValue(ctx, "repo", repo)
+ ctx := context.WithValue(ctx, "repo", safeRepoURL(repo))
repo, err := s.ensureRepoInfoCache(ctx, repo, &unitErrorReporter{reporter})
if err != nil {
@@ -591,7 +620,7 @@ func (s *Source) ensureRepoInfoCache(ctx context.Context, repo string, reporter
if _, ok := s.repoInfoCache.get(repo); ok {
return repo, nil
}
- ctx.Logger().V(2).Info("Caching repository info")
+ ctx.Logger().V(2).Info("Caching repository info", "repo", safeRepoURL(repo))
_, urlParts, err := getRepoURLParts(repo)
if err != nil {
@@ -743,7 +772,6 @@ func (s *Source) enumerateWithToken(ctx context.Context, isGithubEnterprise bool
}
if s.conn.ScanUsers && len(s.memberCache) > 0 {
- ctx.Logger().Info("Fetching repos for org members", "org_count", s.orgsCache.Count(), "member_count", len(s.memberCache))
s.addReposForMembers(ctx, reporter)
}
}
@@ -770,7 +798,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector,
if err != nil {
return err
}
- ctx.Logger().Info("Scanning repos", "org_members", len(s.memberCache))
+ ctx.Logger().V(1).Info("Scanning repos", "org_members", len(s.memberCache))
if !connector.HasDefaultInstallation() {
// Member repos/gists live outside any installation's own
// listing, so there is no installation token to fetch them
@@ -789,11 +817,18 @@ func (s *Source) enumerateWithApp(ctx context.Context, connector *appConnector,
ctx.Logger().Info("Skipping org member gists: GitHub doesn't allow listing gists with a GitHub App installation token", "org_members", len(s.memberCache))
}
for member := range s.memberCache {
- logger := ctx.Logger().WithValues("member", member)
+ // once the scan is cancelled every fetch fails, so stop here
+ // instead of logging an error for each remaining member
+ if common.IsDone(ctx) {
+ ctx.Logger().V(4).Info("skipping remaining members", "reason", "scan cancelled")
+ return nil
+ }
+ // add the member to the error line below so we know whose fetch failed
+ memberCtx := context.WithValues(ctx, "member", member)
// TODO: Add authenticated user list repo for app token. It does support as per docs but need to test it before we enable it here.
// docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repositories-for-the-authenticated-user
- if err := s.getReposByUser(ctx, member, false, reporter); err != nil {
- logger.Error(err, "error fetching repos by user")
+ if err := s.getReposByUser(ctx, member, false, reporter); err != nil && ctx.Err() == nil {
+ memberCtx.Logger().Error(err, "error fetching repos by user")
}
}
}
@@ -890,9 +925,9 @@ func (s *Source) enumerateAllInstallationRepos(ctx context.Context, connector *a
installID := install.GetID()
account := install.GetAccount().GetLogin()
installCtx := context.WithValues(ctx, "org", account, "installation_id", installID)
- installCtx.Logger().Info("Enumerating repos from GitHub App installation")
+ installCtx.Logger().V(2).Info("Enumerating repos from GitHub App installation")
- client, err := connector.APIClientForInstallation(installID)
+ client, err := connector.APIClientForInstallation(installCtx, installID)
if err != nil {
installCtx.Logger().Error(err, "could not create API client for installation")
if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", installID, err)); err != nil {
@@ -981,7 +1016,7 @@ func (s *Source) mapReposToInstallations(ctx context.Context, connector *appConn
installedRepoKeys := make(map[int64]map[string]struct{}, len(installs))
for _, install := range installs {
installID := install.GetID()
- client, err := connector.APIClientForInstallation(installID)
+ client, err := connector.APIClientForInstallation(ctx, installID)
if err != nil {
errs = errors.Join(errs, fmt.Errorf("could not create API client for installation %d: %w", installID, err))
continue
@@ -1199,8 +1234,9 @@ func (s *Source) mapRemainingAccessibleRepos(
continue
}
+ // requested.original is the repo as the user wrote it, so strip any token
ctx.Logger().Info("repo is not owned by any app installation; scanning with the default installation token",
- "repo", requested.original)
+ "repo", safeRepoURL(requested.original))
deleteRepoMappingRequest(wantedRepos, requested)
s.recordRepoInstallation(connector, connector.installationID, repo, requested)
break
@@ -1314,18 +1350,25 @@ func (s *Source) repoURLFromTargetLink(link string) (string, error) {
func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error {
var scannedCount uint64 = 1
- ctx.Logger().V(2).Info("Found repos to scan", "count", len(s.repos))
+ ctx.Logger().V(1).Info("Found repos to scan", "count", len(s.repos))
// If there is resume information available, limit this scan to only the repos that still need scanning.
reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
+ // repos finished by an earlier, interrupted run are left out, so say how many
+ if skipped := len(s.repos) - len(reposToScan); skipped > 0 {
+ ctx.Logger().V(1).Info("resuming scan; skipping repos already scanned", "skipped", skipped, "remaining", len(reposToScan))
+ }
s.repos = reposToScan
for i, repoURL := range s.repos {
s.jobPool.Go(func() error {
+ // repos the user listed in the config are used as-is and may hold a
+ // token, so only the cleaned URL goes into logs
if common.IsDone(ctx) {
+ ctx.Logger().V(4).Info("skipping repo", "repo", safeRepoURL(repoURL), "reason", "scan cancelled")
return nil
}
- ctx := context.WithValue(ctx, "repo", repoURL)
+ ctx := context.WithValue(ctx, "repo", safeRepoURL(repoURL))
// TODO: set progress complete is being called concurrently with i
s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
@@ -1373,23 +1416,35 @@ func (s *Source) scanRepo(ctx context.Context, repoURL string, reporter sources.
}
// Scan the wiki, if enabled, and the repo has one.
- if s.conn.IncludeWikis && repoInfo.hasWiki && s.wikiIsReachable(ctx, repoURL) {
+ // The checks are split up only so each skip can say why.
+ switch {
+ case !s.conn.IncludeWikis:
+ // wikis are off for the whole source, nothing to log per repo
+ case !repoInfo.hasWiki:
+ ctx.Logger().V(4).Info("skipping wiki", "reason", "repo has no wiki")
+ case !s.wikiIsReachable(ctx, repoURL):
+ ctx.Logger().V(4).Info("skipping wiki", "reason", "wiki is not reachable")
+ default:
wikiURL, ok := wikiCloneURLForRepoInfo(repoURL, repoInfo)
- if ok {
- wikiCtx := context.WithValue(ctx, "repo", wikiURL)
+ if !ok {
+ ctx.Logger().V(4).Info("skipping wiki", "reason", "could not build wiki clone URL")
+ break
+ }
+ wikiCtx := context.WithValue(ctx, "repo", safeRepoURL(wikiURL))
- _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)
- if err != nil {
- // Ignore "Repository not found" errors.
- // It's common for GitHub's API to say a repo has a wiki when it doesn't.
- if !strings.Contains(err.Error(), "not found") {
- if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil {
- return err
- }
+ _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, reporter)
+ if err != nil {
+ // Ignore "Repository not found" errors.
+ // It's common for GitHub's API to say a repo has a wiki when it doesn't.
+ if strings.Contains(err.Error(), "not found") {
+ wikiCtx.Logger().V(4).Info("skipping wiki", "reason", "GitHub says the wiki was not found")
+ } else {
+ if err := reporter.ChunkErr(ctx, fmt.Errorf("error scanning wiki: %w", err)); err != nil {
+ return err
}
-
- // Don't return, it still might be possible to scan comments.
}
+
+ // Don't return, it still might be possible to scan comments.
}
}
@@ -1517,7 +1572,7 @@ func (s *Source) handleRateLimit(ctx context.Context, errIn error, reporters ...
if retryAfter > 0 {
retryAfter = retryAfter + jitter
rateLimitResumeTime = now.Add(retryAfter)
- ctx.Logger().Info(fmt.Sprintf("exceeded %s rate limit", limitType), "retry_after", retryAfter.String(), "resume_time", rateLimitResumeTime.Format(time.RFC3339))
+ ctx.Logger().Info("exceeded rate limit", "limit_type", limitType, "retry_after", retryAfter.String(), "resume_time", rateLimitResumeTime.Format(time.RFC3339))
// Only report the error if a reporter was provided
for _, reporter := range reporters {
_ = reporter.Err(ctx, fmt.Errorf("exceeded %s rate limit", limitType))
@@ -1567,13 +1622,21 @@ func (s *Source) handleRateLimitWithChunkReporter(ctx context.Context, reporter
}
func (s *Source) addReposForMembers(ctx context.Context, reporter sources.UnitReporter) {
- ctx.Logger().Info("Fetching repos from members", "members", len(s.memberCache))
+ ctx.Logger().V(1).Info("Fetching repos from members", "members", len(s.memberCache), "org_count", s.orgsCache.Count())
for member := range s.memberCache {
- if err := s.addUserGistsToCache(ctx, member, reporter); err != nil {
- ctx.Logger().Info("Unable to fetch gists by user", "user", member, "error", err)
+ // once the scan is cancelled every fetch fails, so stop here
+ // instead of logging an error for each remaining member
+ if common.IsDone(ctx) {
+ ctx.Logger().V(4).Info("skipping remaining members", "reason", "scan cancelled")
+ return
+ }
+ // a fetch that was running when the scan got cancelled fails too,
+ // that is not an error worth logging
+ if err := s.addUserGistsToCache(ctx, member, reporter); err != nil && ctx.Err() == nil {
+ ctx.Logger().Error(err, "Unable to fetch gists by user", "user", member)
}
- if err := s.getReposByUser(ctx, member, false, reporter); err != nil {
- ctx.Logger().Info("Unable to fetch repos by user", "user", member, "error", err)
+ if err := s.getReposByUser(ctx, member, false, reporter); err != nil && ctx.Err() == nil {
+ ctx.Logger().Error(err, "Unable to fetch repos by user", "user", member)
}
}
}
@@ -1586,7 +1649,8 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporter
}
gistOpts := &github.GistListOptions{}
- logger := ctx.Logger().WithValues("user", user)
+ // every log line below is about this user
+ ctx = context.WithValues(ctx, "user", user)
for {
gists, res, err := s.connector.APIClient().Gists.List(ctx, user, gistOpts)
@@ -1608,7 +1672,8 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string, reporter
if res == nil || res.NextPage == 0 {
break
}
- logger.V(2).Info("Listed gists", "page", gistOpts.Page, "last_page", res.LastPage)
+ // one line per page, with how many API calls we have left before the rate limit
+ ctx.Logger().V(4).Info("Listed gists", "page", gistOpts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)
gistOpts.Page = res.NextPage
}
return nil
@@ -1631,7 +1696,7 @@ func (s *Source) addMembersByApp(ctx context.Context, connector *appConnector, r
// Create a per-installation API client so that the token is scoped to
// this org. GitHub App installation tokens only bypass IP allowlists for
// their own org; using a cross-org token causes 403s.
- client, err := connector.APIClientForInstallation(install.GetID())
+ client, err := connector.APIClientForInstallation(installCtx, install.GetID())
if err != nil {
installCtx.Logger().Error(err, "could not create API client for installation")
if err := reporter.UnitErr(installCtx, fmt.Errorf("could not create API client for installation %d: %w", install.GetID(), err)); err != nil {
@@ -1678,7 +1743,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context, reporter sources.UnitRep
}
lastOrgID := *orgs[len(orgs)-1].ID
- ctx.Logger().V(2).Info(fmt.Sprintf("listed organization IDs %d through %d", orgOpts.Since, lastOrgID))
+ ctx.Logger().V(4).Info("listed organization IDs", "from_id", orgOpts.Since, "to_id", lastOrgID)
orgOpts.Since = lastOrgID
for _, org := range orgs {
@@ -1701,18 +1766,20 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string, reporter source
orgOpts := &github.ListOptions{
PerPage: defaultPagination,
}
- logger := ctx.Logger().WithValues("user", user)
+ // every log line below is about this user
+ ctx = context.WithValues(ctx, "user", user)
for {
orgs, resp, err := s.connector.APIClient().Organizations.List(ctx, "", orgOpts)
if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
continue
}
if err != nil {
- logger.Error(err, "Could not list organizations")
+ ctx.Logger().Error(err, "Could not list organizations")
return
}
- logger.V(2).Info("Listed orgs", "page", orgOpts.Page, "last_page", resp.LastPage)
+ // one line per page, with how many API calls we have left before the rate limit
+ ctx.Logger().V(4).Info("Listed orgs", "page", orgOpts.Page, "last_page", resp.LastPage, "rate_remaining", resp.Rate.Remaining)
for _, org := range orgs {
if org.Login == nil {
continue
@@ -1738,7 +1805,8 @@ func (s *Source) addMembersByOrgWithClient(ctx context.Context, client *github.C
},
}
- logger := ctx.Logger().WithValues("org", org)
+ // every log line below is about this org
+ ctx = context.WithValues(ctx, "org", org)
for {
members, res, err := client.Organizations.ListMembers(ctx, org, opts)
if s.handleRateLimitWithUnitReporter(ctx, reporter, err) {
@@ -1751,7 +1819,8 @@ func (s *Source) addMembersByOrgWithClient(ctx context.Context, client *github.C
return fmt.Errorf("organization (%q) had 0 members: account may not have access to list organization members", org)
}
- logger.V(2).Info("Listed members", "page", opts.Page, "last_page", res.LastPage)
+ // one line per page, with how many API calls we have left before the rate limit
+ ctx.Logger().V(4).Info("Listed members", "page", opts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)
for _, m := range members {
usr := m.Login
if usr == nil || *usr == "" {
@@ -2614,7 +2683,7 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget,
if meta.GetFile() == "" && meta.GetCommit() != "" {
ctx := context.WithValues(ctx, "commit_hash", meta.GetCommit())
- ctx.Logger().V(2).Info("secret metadata has no file; scanning commit metadata instead")
+ ctx.Logger().V(4).Info("secret metadata has no file; scanning commit metadata instead")
return s.scanCommitMetadata(ctx, apiClient, segments[1], segments[2], meta, &chunkSkel, reporter)
}
@@ -2777,7 +2846,7 @@ func (s *Source) scanCommitMetadata(ctx context.Context, apiClient *github.Clien
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
repoURL, kind := unit.SourceUnitID()
- ctx = context.WithValue(ctx, "repo", repoURL)
+ ctx = context.WithValue(ctx, "repo", safeRepoURL(repoURL))
// If the unit has an installation ID, use it. Fetching it is slow,
// specifically it stalls large multi-org scans on rate limits (INT-790).
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index e71c2f289..b00ba3908 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -19,16 +19,21 @@ import (
"testing"
"time"
+ "github.com/go-logr/zapr"
"github.com/google/go-cmp/cmp"
"github.com/google/go-github/v67/github"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
"gopkg.in/h2non/gock.v1"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log/logtest"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
@@ -377,7 +382,7 @@ func TestAppConnector_EnterpriseBaseURL(t *testing.T) {
privateKey := createPrivateKey()
enterpriseEndpoint := "https://api.example.ghe.com"
- connector, err := NewAppConnector(enterpriseEndpoint, &credentialspb.GitHubApp{
+ connector, err := NewAppConnector(context.Background(), enterpriseEndpoint, &credentialspb.GitHubApp{
PrivateKey: privateKey,
InstallationId: "1337",
AppId: "4141",
@@ -3047,3 +3052,192 @@ func TestCreateAPIClient_CloudGitHub(t *testing.T) {
t.Errorf("expected https://api.github.com/, got: %s", client.BaseURL.String())
}
}
+
+func TestEnumerateWithToken_DebugLogging(t *testing.T) {
+ const testToken = "ghp_TESTTOKENdonotlog1234567890abcdef"
+
+ // mock one page of repos: one to scan, plus one of each kind we skip
+ mockAPI := func() {
+ gock.New("https://api.github.com").
+ Get("/user").
+ Reply(200).
+ JSON(map[string]string{"login": "test-user"})
+
+ gock.New("https://api.github.com").
+ Get("/user/repos").
+ Reply(200).
+ JSON([]map[string]any{
+ {"full_name": "test-user/kept-repo", "clone_url": "https://github.com/test-user/kept-repo.git"},
+ {"full_name": "test-user/forked-repo", "clone_url": "https://github.com/test-user/forked-repo.git", "fork": true},
+ {"full_name": "test-user/archived-repo", "clone_url": "https://github.com/test-user/archived-repo.git", "archived": true},
+ {"full_name": "test-user/ignored-repo", "clone_url": "https://github.com/test-user/ignored-repo.git"},
+ })
+
+ gock.New("https://api.github.com").
+ Get("/user/orgs").
+ Reply(200).
+ JSON(`[]`)
+
+ gock.New("https://api.github.com").
+ Get("/users/test-user/gists").
+ Reply(200).
+ JSON(`[]`)
+ }
+
+ // init the source with the given context and list the repos
+ runEnumerate := func(t *testing.T, ctx context.Context) *Source {
+ s, conn := createTestSource(&sourcespb.GitHub{
+ Endpoint: "https://api.github.com",
+ Credential: &sourcespb.GitHub_Token{Token: testToken},
+ ExcludeArchived: true,
+ IgnoreRepos: []string{"test-user/ignored-repo"},
+ })
+ assert.NoError(t, s.Init(ctx, "test - github", 1, 1337, false, conn, 1))
+ gock.InterceptClient(s.connector.APIClient().Client())
+
+ assert.NoError(t, s.enumerateWithToken(ctx, false, noopReporter()))
+ assert.True(t, gock.IsDone())
+ return s
+ }
+
+ t.Run("debug on", func(t *testing.T) {
+ defer gock.Off()
+ mockAPI()
+
+ ctx, logs := logtest.DebugContext()
+ ctx = context.WithValues(ctx, "source_name", "test - github", "job_id", 1)
+ s := runEnumerate(t, ctx)
+ assert.True(t, s.filteredRepoCache.Exists("test-user/kept-repo"))
+
+ // the config line from Init shows the auth type and filters, not the token
+ initLines := logs.FilterMessage("github source initialized").All()
+ if assert.Len(t, initLines, 1) {
+ fields := initLines[0].ContextMap()
+ assert.Equal(t, "*sourcespb.GitHub_Token", fields["auth_type"])
+ assert.Equal(t, int64(1), fields["ignore_repo_count"])
+ assert.Equal(t, true, fields["exclude_archived"])
+ }
+
+ // the page line and one skip line for each skipped repo, with the reason
+ assert.Equal(t, 1, logs.FilterMessage("Listed repos").Len())
+ reasons := map[string]string{}
+ for _, entry := range logs.FilterMessage("skipping repository").All() {
+ fields := entry.ContextMap()
+ reasons[fields["repo"].(string)] = fields["reason"].(string)
+ }
+ assert.Equal(t, map[string]string{
+ "test-user/forked-repo": "forks are not included",
+ "test-user/archived-repo": "archived repos are excluded",
+ "test-user/ignored-repo": "excluded by include or ignore filter",
+ }, reasons)
+
+ // every line came from the context logger, so it has the source fields,
+ // and no line contains the token
+ for _, entry := range logs.All() {
+ fields := entry.ContextMap()
+ assert.Contains(t, fields, "source_name", "line %q", entry.Message)
+ assert.Contains(t, fields, "job_id", "line %q", entry.Message)
+ assert.NotContains(t, entry.Message, testToken)
+ assert.NotContains(t, fmt.Sprint(fields), testToken)
+ }
+ })
+
+ t.Run("debug off", func(t *testing.T) {
+ defer gock.Off()
+ mockAPI()
+
+ core, logs := observer.New(zapcore.InfoLevel)
+ ctx := context.WithLogger(context.Background(), zapr.NewLogger(zap.New(core)))
+ runEnumerate(t, ctx)
+
+ assert.Zero(t, logs.FilterMessage("Listed repos").Len())
+ assert.Zero(t, logs.FilterMessage("skipping repository").Len())
+ })
+}
+
+func TestScan_LogsResumeAndCancelSkips(t *testing.T) {
+ const testToken = "ghp_TESTTOKENinrepourl1234567890"
+
+ s := initTestSource(&sourcespb.GitHub{
+ Endpoint: "https://api.github.com",
+ Credential: &sourcespb.GitHub_Token{Token: "token"},
+ })
+ s.repos = []string{
+ "https://github.com/org/a.git",
+ "https://github.com/org/b.git",
+ // a repo from the config with a token in it, which must never be logged
+ "https://user:" + testToken + "@github.com/org/c.git",
+ }
+ // pretend an earlier run stopped while scanning b, so a is already done
+ s.SetProgressComplete(0, 3, "", sources.EncodeResumeInfo([]string{"https://github.com/org/b.git"}))
+
+ // cancel before the scan starts, so no repo is cloned and each one is
+ // skipped with a log line instead
+ debugCtx, logs := logtest.DebugContext()
+ ctx, cancel := context.WithCancel(debugCtx)
+ cancel()
+
+ assert.NoError(t, s.scan(ctx, &collectingReporter{}))
+
+ resumeLines := logs.FilterMessage("resuming scan; skipping repos already scanned").All()
+ if assert.Len(t, resumeLines, 1) {
+ assert.Equal(t, int64(1), resumeLines[0].ContextMap()["skipped"])
+ assert.Equal(t, int64(2), resumeLines[0].ContextMap()["remaining"])
+ }
+
+ var cancelled []string
+ for _, entry := range logs.FilterMessage("skipping repo").All() {
+ assert.Equal(t, "scan cancelled", entry.ContextMap()["reason"])
+ cancelled = append(cancelled, entry.ContextMap()["repo"].(string))
+ }
+ assert.ElementsMatch(t, []string{"https://github.com/org/b.git", "https://github.com/org/c.git"}, cancelled)
+
+ for _, entry := range logs.All() {
+ assert.NotContains(t, entry.Message, testToken)
+ assert.NotContains(t, fmt.Sprint(entry.ContextMap()), testToken)
+ }
+}
+
+func TestInit_InvalidRepoErrorDoesNotLogToken(t *testing.T) {
+ const testToken = "ghp_TESTTOKENinvalidrepo1234567890"
+
+ // a repo from the config with a token in it and no repo name, so it fails
+ // to normalize and the error repeats the URL
+ s, conn := createTestSource(&sourcespb.GitHub{
+ Endpoint: "https://api.github.com",
+ Credential: &sourcespb.GitHub_Token{Token: "token"},
+ Repositories: []string{"https://" + testToken + "@github.com/org"},
+ })
+
+ ctx, logs := logtest.DebugContext()
+ assert.NoError(t, s.Init(ctx, "test - github", 0, 1337, false, conn, 1))
+
+ errLines := logs.FilterMessage("invalid repository").All()
+ if assert.Len(t, errLines, 1) {
+ fields := errLines[0].ContextMap()
+ assert.Equal(t, "https://github.com/org", fields["repo"])
+ assert.Contains(t, fields["error"], "missing the repo name")
+ }
+ for _, entry := range logs.All() {
+ assert.NotContains(t, entry.Message, testToken)
+ assert.NotContains(t, fmt.Sprint(entry.ContextMap()), testToken)
+ }
+}
+
+func TestAddReposForMembers_StopsOnCancel(t *testing.T) {
+ s := initTestSource(&sourcespb.GitHub{
+ Endpoint: "https://api.github.com",
+ Credential: &sourcespb.GitHub_Token{Token: "token"},
+ })
+ s.memberCache = map[string]struct{}{"member1": {}, "member2": {}}
+
+ // cancel before the loop starts, so no member is fetched and no error is logged
+ debugCtx, logs := logtest.DebugContext()
+ ctx, cancel := context.WithCancel(debugCtx)
+ cancel()
+
+ s.addReposForMembers(ctx, noopReporter())
+
+ assert.Equal(t, 1, logs.FilterMessage("skipping remaining members").Len())
+ assert.Zero(t, logs.FilterLevelExact(zapcore.ErrorLevel).Len())
+}
diff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.go
index 4efb21c17..6af4c1cad 100644
--- a/pkg/sources/github/graphql.go
+++ b/pkg/sources/github/graphql.go
@@ -45,7 +45,7 @@ func (s *Source) processIssuesWithComments(
totalIssues += len(query.GetIssues())
- ctx.Logger().V(5).Info("Scanning Issues",
+ ctx.Logger().V(4).Info("Scanning Issues",
"total_issues", len(query.GetIssues()))
if err := s.chunkGraphqlIssues(ctx, repoInfo, query.Repository.Issues.Nodes, reporter); err != nil {
@@ -54,7 +54,7 @@ func (s *Source) processIssuesWithComments(
// process each issue comments
for _, issue := range query.GetIssues() {
- ctx.Logger().V(5).Info("Scanning Issue Comments",
+ ctx.Logger().V(3).Info("Scanning Issue Comments",
"issue_id", issue.Number,
"total_comments", len(issue.GetIssueComments()),
)
@@ -84,7 +84,7 @@ func (s *Source) processIssuesWithComments(
return fmt.Errorf("error fetching issue: %w", err)
}
- ctx.Logger().V(5).Info("Scanning additional issue comments",
+ ctx.Logger().V(4).Info("Scanning additional issue comments",
"issue_id", issue.Number,
"total_comments", len(commentsQuery.GetIssueComments()))
@@ -137,7 +137,7 @@ func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githu
totalPRs += len(query.GetPullRequests())
- ctx.Logger().V(5).Info("Scanning pull requests",
+ ctx.Logger().V(4).Info("Scanning pull requests",
"total_pull_requests", len(query.GetPullRequests()))
if err := s.chunkGraphqlPullRequests(ctx, repoInfo, query.GetPullRequests(), reporter); err != nil {
@@ -146,7 +146,7 @@ func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githu
// process each pr comments
for _, pr := range query.GetPullRequests() {
- ctx.Logger().V(5).Info("Scanning pull request comments",
+ ctx.Logger().V(3).Info("Scanning pull request comments",
"pull_request_no", pr.Number,
"total_comments", len(pr.Comments.Nodes))
@@ -175,7 +175,7 @@ func (s *Source) processPRWithComments(ctx context.Context, graphqlClient *githu
return fmt.Errorf("error fetching pull request with comments: %w", err)
}
- ctx.Logger().V(5).Info("Scanning additional comments",
+ ctx.Logger().V(4).Info("Scanning additional comments",
"pull_request_no", pr.Number,
"total_comments", len(commentQuery.GetPRComments()))
@@ -247,7 +247,7 @@ func (s *Source) processReviewThreads(ctx context.Context, graphqlClient *github
if len(threadIDs) > 0 {
// process in batches of max 100
for _, batch := range chunkIDs(threadIDs, 100) {
- ctx.Logger().V(5).Info("Processing Thread comments in Batches", "batch_length", len(batch))
+ ctx.Logger().V(4).Info("Processing Thread comments in Batches", "batch_length", len(batch))
// fetch comments for the batch of threads
if err := s.fetchThreadComments(ctx, graphqlClient, batch, repoInfo, reporter, cutoffTime); err != nil {
return fmt.Errorf("error fetching thread review comments: %w", err)
diff --git a/pkg/sources/github/pagination.go b/pkg/sources/github/pagination.go
index 998296020..8ab23a656 100644
--- a/pkg/sources/github/pagination.go
+++ b/pkg/sources/github/pagination.go
@@ -251,7 +251,7 @@ func walkUpdatedSince[T any](ctx context.Context, p pager, w sinceWalk[T]) (int,
}
windowStart = latest
- ctx.Logger().V(2).Info("listing capped, continuing from last update time",
+ ctx.Logger().V(4).Info("listing capped, continuing from last update time",
"listing", w.name, "since", windowStart.UTC().Format(time.RFC3339), "items_so_far", total)
}
}
@@ -305,8 +305,8 @@ func fetchWithRetry[T any](ctx context.Context, p pager, fetch func() ([]T, *git
}
delay := transientBackoff(transientRetries)
transientRetries++
- ctx.Logger().V(2).Info("retrying GitHub API request after transient error",
- "listing", p.listing, "retry", transientRetries, "max_retries", maxTransientRetries, "retry_in", delay.String(), "error", err.Error())
+ ctx.Logger().V(4).Info("retrying GitHub API request after transient error",
+ "listing", p.listing, "retry", transientRetries, "max_retries", maxTransientRetries, "retry_in", delay.String(), "reason", err.Error())
if !p.wait(ctx, delay) {
return items, resp, err
}
diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go
index 5c62f2ede..e77262b82 100644
--- a/pkg/sources/github/repo.go
+++ b/pkg/sources/github/repo.go
@@ -257,7 +257,8 @@ func isGitHubListingDisabled(err error) bool {
// processRepos processes repositories from a source, handling pagination and rate limits.
func (s *Source) processRepos(ctx context.Context, target string, reporter sources.UnitReporter, listRepos repoLister, listOpts repoListOptions) error {
- logger := ctx.Logger().WithValues("target", target)
+ // every log line below is about this org or user
+ ctx = context.WithValues(ctx, "target", target)
opts := listOpts.getListOptions()
var (
@@ -276,17 +277,19 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
return err
}
- ctx.Logger().V(2).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage)
+ // one line per page, with how many API calls we have left before the rate limit
+ ctx.Logger().V(4).Info("Listed repos", "page", opts.Page, "last_page", res.LastPage, "rate_remaining", res.Rate.Remaining)
for _, r := range someRepos {
isFork, isArchived := r.GetFork(), r.GetArchived()
if isFork && !s.conn.IncludeForks {
+ ctx.Logger().V(4).Info("skipping repository", "repo", r.GetFullName(), "reason", "forks are not included")
continue
}
if isArchived && s.conn.ExcludeArchived {
numArchivedSkipped++
- logger.V(3).Info("skipping archived repository", "repo", r.GetFullName())
+ ctx.Logger().V(4).Info("skipping repository", "repo", r.GetFullName(), "reason", "archived repos are excluded")
continue
}
@@ -309,6 +312,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
// Check if we should process this repository based on the filter
if !s.filteredRepoCache.wantRepo(repoName) {
+ ctx.Logger().V(4).Info("skipping repository", "repo", repoName, "reason", "excluded by include or ignore filter")
continue
}
@@ -334,7 +338,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
return err
}
- logger.V(3).Info("repo attributes", "name", repoName, "kb_size", r.GetSize(), "repo_url", repoURL)
+ ctx.Logger().V(2).Info("repo attributes", "name", repoName, "kb_size", r.GetSize(), "repo_url", repoURL)
}
if res.NextPage == 0 {
@@ -344,7 +348,7 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
}
// final logging of repository stats.
- logger.V(2).Info("found repos", "total", numRepos, "num_forks", numForks, "num_archived", numArchived, "num_archived_skipped", numArchivedSkipped, "num_orgs", len(uniqueOrgs))
+ ctx.Logger().V(2).Info("found repos", "total", numRepos, "num_forks", numForks, "num_archived", numArchived, "num_archived_skipped", numArchivedSkipped, "num_orgs", len(uniqueOrgs))
githubOrgsEnumerated.WithLabelValues(s.name).Add(float64(len(uniqueOrgs)))
return nil
diff --git a/pkg/sources/github/urls.go b/pkg/sources/github/urls.go
index 7b8ee7029..ead3d101a 100644
--- a/pkg/sources/github/urls.go
+++ b/pkg/sources/github/urls.go
@@ -6,6 +6,79 @@ import (
"strings"
)
+// safeRepoURL removes any username or password from a repo URL so it can be
+// logged.
+func safeRepoURL(repo string) string {
+ // a normal URL like "https://user:pass@host/org/repo": let url.Parse find
+ // the user info and drop it
+ u, err := url.Parse(repo)
+ if err == nil && u.Host != "" {
+ u.User = nil
+ return u.String()
+ }
+
+ // anything else is a repo name like "org/repo", or a URL that url.Parse
+ // rejects, so drop the user info by hand
+ if prefix, _, rest, found := splitUserInfo(repo); found {
+ return prefix + rest
+ }
+ return repo
+}
+
+// splitUserInfo splits a repo that url.Parse rejects, like an SCP-style URL
+// "user:pass@host:org/repo" or a mix like "https://token@host:org/repo", into
+// the scheme prefix ("https://" or ""), the user info and the rest. The user
+// info always sits before an "@" in the part before the first "/" after any
+// scheme. found is false when there is no such "@".
+func splitUserInfo(repo string) (prefix, userInfo, rest string, found bool) {
+ scheme, afterScheme, hasScheme := strings.Cut(repo, "://")
+ if hasScheme {
+ prefix = scheme + "://"
+ } else {
+ afterScheme = repo
+ }
+ hostPart, _, _ := strings.Cut(afterScheme, "/")
+ at := strings.LastIndex(hostPart, "@")
+ if at < 0 {
+ return prefix, "", afterScheme, false
+ }
+ return prefix, afterScheme[:at], afterScheme[at+1:], true
+}
+
+// repoCredential returns the secret in a repo URL's user info, or "" if it has
+// none, so it can be redacted from logs. That is the password, or for an
+// http(s) URL with no password the username, since "https://TOKEN@host" puts
+// the token there. Other usernames are not secret, and redacting one like
+// "git" would hide that word in every log line.
+func repoCredential(repo string) string {
+ var prefix, username, password string
+ if u, err := url.Parse(repo); err == nil && u.Host != "" {
+ if u.User == nil {
+ return ""
+ }
+ prefix = u.Scheme + "://"
+ username = u.User.Username()
+ password, _ = u.User.Password()
+ } else {
+ var userInfo string
+ var found bool
+ prefix, userInfo, _, found = splitUserInfo(repo)
+ if !found {
+ return ""
+ }
+ username, password, _ = strings.Cut(userInfo, ":")
+ }
+
+ // a password can be present but empty, as in "https://TOKEN:@host"
+ if password != "" {
+ return password
+ }
+ if prefix == "http://" || prefix == "https://" {
+ return username
+ }
+ return ""
+}
+
func wikiCloneURLForRepo(repoURL string) (string, bool) {
if !strings.HasSuffix(repoURL, ".git") || strings.HasSuffix(repoURL, ".wiki.git") {
return "", false
diff --git a/pkg/sources/github/urls_test.go b/pkg/sources/github/urls_test.go
index 85bc50ee1..c1b8c82d5 100644
--- a/pkg/sources/github/urls_test.go
+++ b/pkg/sources/github/urls_test.go
@@ -171,3 +171,42 @@ func TestIsWikiLink(t *testing.T) {
assert.False(t, isWikiLink("https://github.com/owner/repo/blob/main/path/to/file"))
assert.False(t, isWikiLink("not a url"))
}
+
+func TestSafeRepoURL(t *testing.T) {
+ tests := map[string]string{
+ "https://user:[email protected]/org/repo.git": "https://github.com/org/repo.git",
+ "https://[email protected]/org/repo.git": "https://github.com/org/repo.git",
+ "https://github.com/org/repo.git": "https://github.com/org/repo.git",
+ "org/repo": "org/repo",
+ "[email protected]:org/repo.git": "github.com:org/repo.git",
+ "user:[email protected]:org/repo.git": "github.com:org/repo.git",
+ // mixed HTTPS and SCP style, which url.Parse rejects
+ "https://[email protected]:org/repo.git": "https://github.com:org/repo.git",
+ "https://user:[email protected]:org/repo.git": "https://github.com:org/repo.git",
+ }
+ for input, want := range tests {
+ assert.Equal(t, want, safeRepoURL(input), input)
+ }
+}
+
+func TestRepoCredential(t *testing.T) {
+ tests := map[string]string{
+ "https://user:[email protected]/org/repo.git": "ghp_secret",
+ "https://[email protected]/org/repo.git": "ghp_secret",
+ "ssh://git:[email protected]/org/repo.git": "ghp_secret",
+ // a plain SSH user is not a secret
+ "ssh://[email protected]/org/repo.git": "",
+ "https://github.com/org/repo.git": "",
+ "org/repo": "",
+ "[email protected]:org/repo.git": "",
+ // the password is present but empty, so the username is the token
+ "https://ghp_secret:@github.com/org/repo.git": "ghp_secret",
+ // SCP style and mixed URLs, which url.Parse rejects
+ "user:[email protected]:org/repo.git": "ghp_secret",
+ "https://[email protected]:org/repo.git": "ghp_secret",
+ "https://user:[email protected]:org/repo.git": "ghp_secret",
+ }
+ for input, want := range tests {
+ assert.Equal(t, want, repoCredential(input), input)
+ }
+}