Skip to content

Find, verify, and analyze leaked credentials

correct logging output for github comments and add oss flags (#1632)

Zubair Khandb89e345d758
diff --git a/main.go b/main.go
index c680b7c08..c31a1d1ac 100644
--- a/main.go
+++ b/main.go
@@ -73,17 +73,20 @@ var (
_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
- githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
- githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
- githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
- githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
- githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
- githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
- githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
- githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
- githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
- githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
- githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+ githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
+ githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+ githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+ githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+ githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
+ githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
+ githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+ githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
+ githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
+ githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+ githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+ githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool()
+ githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool()
+ githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
// TODO: Add more GitLab options
@@ -409,16 +412,19 @@ func run(state overseer.State) {
}
cfg := sources.GithubConfig{
- Endpoint: *githubScanEndpoint,
- Token: *githubScanToken,
- IncludeForks: *githubIncludeForks,
- IncludeMembers: *githubIncludeMembers,
- Concurrency: *concurrency,
- ExcludeRepos: *githubExcludeRepos,
- IncludeRepos: *githubIncludeRepos,
- Repos: *githubScanRepos,
- Orgs: *githubScanOrgs,
- Filter: filter,
+ Endpoint: *githubScanEndpoint,
+ Token: *githubScanToken,
+ IncludeForks: *githubIncludeForks,
+ IncludeMembers: *githubIncludeMembers,
+ Concurrency: *concurrency,
+ ExcludeRepos: *githubExcludeRepos,
+ IncludeRepos: *githubIncludeRepos,
+ Repos: *githubScanRepos,
+ Orgs: *githubScanOrgs,
+ IncludeIssueComments: *githubScanIssueComments,
+ IncludePullRequestComments: *githubScanPRComments,
+ IncludeGistComments: *githubScanGistComments,
+ Filter: filter,
}
if err := e.ScanGitHub(ctx, cfg); err != nil {
logFatal(err, "Failed to scan Github.")
diff --git a/pkg/engine/github.go b/pkg/engine/github.go
index 0442c4756..4e83acbdd 100644
--- a/pkg/engine/github.go
+++ b/pkg/engine/github.go
@@ -15,12 +15,15 @@ import (
// ScanGitHub scans Github with the provided options.
func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
connection := sourcespb.GitHub{
- Endpoint: c.Endpoint,
- Organizations: c.Orgs,
- Repositories: c.Repos,
- ScanUsers: c.IncludeMembers,
- IgnoreRepos: c.ExcludeRepos,
- IncludeRepos: c.IncludeRepos,
+ Endpoint: c.Endpoint,
+ Organizations: c.Orgs,
+ Repositories: c.Repos,
+ ScanUsers: c.IncludeMembers,
+ IgnoreRepos: c.ExcludeRepos,
+ IncludeRepos: c.IncludeRepos,
+ IncludeIssueComments: c.IncludeIssueComments,
+ IncludePullRequestComments: c.IncludePullRequestComments,
+ IncludeGistComments: c.IncludeGistComments,
}
if len(c.Token) > 0 {
connection.Credential = &sourcespb.GitHub_Token{
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index c4b0df851..dea631e5c 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -978,8 +978,6 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
}
func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
- s.log.Info("scanning comments", "repository", repoPath)
-
// Support ssh and https URLs
repoURL, err := git.GitURLParse(repoPath)
if err != nil {
@@ -988,6 +986,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
trimmedURL := removeURLAndSplit(repoURL.String())
if repoURL.Host == "gist.github.com" && s.includeGistComments {
+ s.log.Info("scanning github gist comments", "repository", repoPath)
// GitHub Gist URL.
var gistId string
if len(trimmedURL) == 2 {
@@ -1036,6 +1035,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
if s.includeIssueComments {
+ s.log.Info("scanning github issue comments", "repository", repoPath)
+
issueOpts := &github.IssueListCommentsOptions{
Sort: &sortType,
Direction: &directionType,
@@ -1070,6 +1071,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
}
if s.includePRComments {
+ s.log.Info("scanning github pull request comments", "repository", repoPath)
+
prOpts := &github.PullRequestListCommentsOptions{
Sort: sortType,
Direction: directionType,
diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go
index 4ecc46e80..af6fa62d1 100644
--- a/pkg/sources/sources.go
+++ b/pkg/sources/sources.go
@@ -172,6 +172,12 @@ type GithubConfig struct {
IncludeRepos []string
// Filter is the filter to use to scan the source.
Filter *common.Filter
+ // IncludeIssueComments indicates whether to include GitHub issue comments in the scan.
+ IncludeIssueComments,
+ // IncludePullRequestComments indicates whether to include GitHub pull request comments in the scan.
+ IncludePullRequestComments,
+ // IncludeGistComments indicates whether to include GitHub gist comments in the scan.
+ IncludeGistComments bool
}
// GitlabConfig defines the optional configuration for a gitlab source.