Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Integrate GitHub source
Dustin Decker03ead2f7ed0f
diff --git a/main.go b/main.goindex b953ab0a2..9790e10ed 100644--- a/main.go+++ b/main.go@@ -9,13 +9,13 @@ import ("runtime""strconv"- "github.com/trufflesecurity/trufflehog/pkg/common"- "github.com/trufflesecurity/trufflehog/pkg/sources/git"-"github.com/sirupsen/logrus"+ kingpin "gopkg.in/alecthomas/kingpin.v2"++ "github.com/trufflesecurity/trufflehog/pkg/common""github.com/trufflesecurity/trufflehog/pkg/decoders""github.com/trufflesecurity/trufflehog/pkg/engine"- kingpin "gopkg.in/alecthomas/kingpin.v2"+ "github.com/trufflesecurity/trufflehog/pkg/sources/git")func main() {@@ -39,23 +39,27 @@ func main() {gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()githubScan := cli.Command("github", "Find credentials in GitHub repositories.")- // githubScanTarget := githubScan.Arg("target", "GitHub target. Can be a repository, user or organization.").Required().String()- // githubScanToken := githubScan.Flag("token", "GitHub token.").String()+ githubScanEndpoint := githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()+ githubScanRepos := githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()+ githubScanOrgs := githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()+ githubScanToken := githubScan.Flag("token", "GitHub token.").String()gitlabScan := cli.Command("gitlab", "Find credentials in GitLab repositories.")// gitlabScanTarget := gitlabScan.Arg("target", "GitLab target. Can be a repository, user or organization.").Required().String()// gitlabScanToken := gitlabScan.Flag("token", "GitLab token.").String()- // bitbucketScan := cli.Command("bitbucket", "Find credentials in Bitbucket repositories.")+ bitbucketScan := cli.Command("bitbucket", "Find credentials in Bitbucket repositories.")// bitbucketScanTarget := bitbucketScan.Arg("target", "Bitbucket target. Can be a repository, user or organization.").Required().String()// bitbucketScanToken := bitbucketScan.Flag("token", "Bitbucket token.").String()- // filesystemScan := cli.Command("filesystem", "Find credentials in filesystem.")+ filesystemScan := cli.Command("filesystem", "Find credentials in a filesystem.")// filesystemScanPath := filesystemScan.Arg("path", "Path to scan.").Required().String()// filesystemScanRecursive := filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()// filesystemScanIncludePaths := filesystemScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()// filesystemScanExcludePaths := filesystemScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()+ s3Scan := cli.Command("s3", "Find credentials in an S3 bucket.")+cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))if *jsonOut {@@ -93,9 +97,21 @@ func main() {logrus.WithError(err).Fatal("Failed to scan git.")}case githubScan.FullCommand():- log.Fatal("github not implemented")+ if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {+ log.Fatal("You must specify at least one organization or repository.")+ }+ err = e.ScanGitHub(ctx, *githubScanEndpoint, *githubScanRepos, *githubScanOrgs, *githubScanToken, filter, *concurrency)+ if err != nil {+ logrus.WithError(err).Fatal("Failed to scan git.")+ }case gitlabScan.FullCommand():log.Fatal("gitlab not implemented")+ case bitbucketScan.FullCommand():+ log.Fatal("bitbucket not implemented")+ case filesystemScan.FullCommand():+ log.Fatal("filesystem not implemented")+ case s3Scan.FullCommand():+ log.Fatal("s3 not implemented")}// deal with the results from e.ResultsChan()diff --git a/pkg/engine/github.go b/pkg/engine/github.gonew file mode 100644index 000000000..3441f9de8--- /dev/null+++ b/pkg/engine/github.go@@ -0,0 +1,49 @@+package engine++import (+ "context"++ "github.com/sirupsen/logrus"+ "google.golang.org/protobuf/proto"+ anypb "google.golang.org/protobuf/types/known/anypb"++ "github.com/trufflesecurity/trufflehog/pkg/common"+ "github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/pkg/sources/github"+)++func (e *Engine) ScanGitHub(ctx context.Context, endpoint string, repos, orgs []string, token string, filter *common.Filter, concurrency int) error {+ source := github.Source{}+ connection := sourcespb.GitHub{+ Endpoint: endpoint,+ Organizations: orgs,+ Repositories: repos,+ }+ if len(token) > 0 {+ connection.Credential = &sourcespb.GitHub_Token{+ Token: token,+ }+ } else {+ connection.Credential = &sourcespb.GitHub_Unauthenticated{}+ }+ var conn anypb.Any+ err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})+ if err != nil {+ logrus.WithError(err).Error("failed to marshal github connection")+ return err+ }+ err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, concurrency)+ if err != nil {+ logrus.WithError(err).Error("failed to initialize github source")+ return err+ }++ go func() {+ err := source.Chunks(ctx, e.ChunksChan())+ if err != nil {+ logrus.WithError(err).Fatal("could not scan github")+ }+ close(e.ChunksChan())+ }()+ return nil+}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex b4a9622a3..f6f8967ab 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -140,11 +140,25 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errcase *sourcespb.GitHub_Unauthenticated:apiClient := github.NewClient(s.httpClient)if len(s.orgs) > 30 {- log.Warn("You may experience rate limiting due with the unauthenticated GitHub api. Consider using the authenticated org user scan feature instead")+ log.Warn("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")}- for _, org := range s.orgs {- org = strings.TrimSpace(org)- if !strings.HasSuffix(org, ".git") {++ if len(s.repos) > 0 {+ for i, repo := range s.repos {+ if !strings.HasSuffix(repo, ".git") {+ if repo, err := giturl.NormalizeGithubRepo(repo); err != nil {+ // This wasn't formatted as expected, let the user know why that might be.+ log.WithError(err).Warnf("Repo not in expected format, attempting to paginate repos instead.")+ } else {+ s.repos[i] = repo+ }+ s.paginateRepos(ctx, apiClient, repo)+ }+ }+ }++ if len(s.orgs) > 0 {+ for _, org := range s.orgs {s.paginateRepos(ctx, apiClient, org)}}@@ -203,6 +217,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) errif err != nil {return errors.New(err)}+ // TODO: this should enumerate an organizations gists too...s.paginateGists(ctx, user.GetLogin(), chunksChan)if !specificScope {diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 3b0010abb..3751468d2 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -11,12 +11,13 @@ import ("github.com/kylelemons/godebug/pretty""github.com/mattn/go-colorable"log "github.com/sirupsen/logrus"+ "google.golang.org/protobuf/types/known/anypb"+"github.com/trufflesecurity/trufflehog/pkg/common""github.com/trufflesecurity/trufflehog/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb""github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb""github.com/trufflesecurity/trufflehog/pkg/sources"- "google.golang.org/protobuf/types/known/anypb")func TestSource_Scan(t *testing.T) {@@ -192,6 +193,29 @@ func TestSource_Scan(t *testing.T) {},wantErr: false,},+ {+ name: "get an unauthenticated repo chunk with no enumeration",+ init: init{+ name: "test source",+ connection: &sourcespb.GitHub{+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},+ Credential: &sourcespb.GitHub_Unauthenticated{},+ },+ },+ wantChunk: &sources.Chunk{+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ SourceName: "test source",+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Github{+ Github: &source_metadatapb.Github{+ Repository: "https://github.com/trufflesecurity/driftwood.git",+ },+ },+ },+ Verify: false,+ },+ wantErr: false,+ },{name: "installed app on org w/ enum",init: init{