Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
correct logging output for github comments and add oss flags (#1632)
Zubair Khandb89e345d758
diff --git a/main.go b/main.goindex c680b7c08..c31a1d1ac 100644--- a/main.go+++ b/main.go@@ -73,17 +73,20 @@ var (_ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()_ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()- githubScan = cli.Command("github", "Find credentials in GitHub repositories.")- githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()- githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()- githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()- githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()- githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()- githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()- githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()- githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()- githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()- githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()+ githubScan = cli.Command("github", "Find credentials in GitHub repositories.")+ githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()+ githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()+ githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()+ githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()+ githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()+ githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()+ githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()+ githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()+ githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()+ githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()+ githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool()+ githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool()+ githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")// TODO: Add more GitLab options@@ -409,16 +412,19 @@ func run(state overseer.State) {}cfg := sources.GithubConfig{- Endpoint: *githubScanEndpoint,- Token: *githubScanToken,- IncludeForks: *githubIncludeForks,- IncludeMembers: *githubIncludeMembers,- Concurrency: *concurrency,- ExcludeRepos: *githubExcludeRepos,- IncludeRepos: *githubIncludeRepos,- Repos: *githubScanRepos,- Orgs: *githubScanOrgs,- Filter: filter,+ Endpoint: *githubScanEndpoint,+ Token: *githubScanToken,+ IncludeForks: *githubIncludeForks,+ IncludeMembers: *githubIncludeMembers,+ Concurrency: *concurrency,+ ExcludeRepos: *githubExcludeRepos,+ IncludeRepos: *githubIncludeRepos,+ Repos: *githubScanRepos,+ Orgs: *githubScanOrgs,+ IncludeIssueComments: *githubScanIssueComments,+ IncludePullRequestComments: *githubScanPRComments,+ IncludeGistComments: *githubScanGistComments,+ Filter: filter,}if err := e.ScanGitHub(ctx, cfg); err != nil {logFatal(err, "Failed to scan Github.")diff --git a/pkg/engine/github.go b/pkg/engine/github.goindex 0442c4756..4e83acbdd 100644--- a/pkg/engine/github.go+++ b/pkg/engine/github.go@@ -15,12 +15,15 @@ import (// ScanGitHub scans Github with the provided options.func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {connection := sourcespb.GitHub{- Endpoint: c.Endpoint,- Organizations: c.Orgs,- Repositories: c.Repos,- ScanUsers: c.IncludeMembers,- IgnoreRepos: c.ExcludeRepos,- IncludeRepos: c.IncludeRepos,+ Endpoint: c.Endpoint,+ Organizations: c.Orgs,+ Repositories: c.Repos,+ ScanUsers: c.IncludeMembers,+ IgnoreRepos: c.ExcludeRepos,+ IncludeRepos: c.IncludeRepos,+ IncludeIssueComments: c.IncludeIssueComments,+ IncludePullRequestComments: c.IncludePullRequestComments,+ IncludeGistComments: c.IncludeGistComments,}if len(c.Token) > 0 {connection.Credential = &sourcespb.GitHub_Token{diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex c4b0df851..dea631e5c 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -978,8 +978,6 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri}func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {- s.log.Info("scanning comments", "repository", repoPath)-// Support ssh and https URLsrepoURL, err := git.GitURLParse(repoPath)if err != nil {@@ -988,6 +986,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan ctrimmedURL := removeURLAndSplit(repoURL.String())if repoURL.Host == "gist.github.com" && s.includeGistComments {+ s.log.Info("scanning github gist comments", "repository", repoPath)// GitHub Gist URL.var gistId stringif len(trimmedURL) == 2 {@@ -1036,6 +1035,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan cif s.includeIssueComments {+ s.log.Info("scanning github issue comments", "repository", repoPath)+issueOpts := &github.IssueListCommentsOptions{Sort: &sortType,Direction: &directionType,@@ -1070,6 +1071,8 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c}if s.includePRComments {+ s.log.Info("scanning github pull request comments", "repository", repoPath)+prOpts := &github.PullRequestListCommentsOptions{Sort: sortType,Direction: directionType,diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex 4ecc46e80..af6fa62d1 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -172,6 +172,12 @@ type GithubConfig struct {IncludeRepos []string// Filter is the filter to use to scan the source.Filter *common.Filter+ // IncludeIssueComments indicates whether to include GitHub issue comments in the scan.+ IncludeIssueComments,+ // IncludePullRequestComments indicates whether to include GitHub pull request comments in the scan.+ IncludePullRequestComments,+ // IncludeGistComments indicates whether to include GitHub gist comments in the scan.+ IncludeGistComments bool}// GitlabConfig defines the optional configuration for a gitlab source.