Skip to content

Find, verify, and analyze leaked credentials

Integrate GitHub source

Dustin Decker03ead2f7ed0f
diff --git a/main.go b/main.go
index b953ab0a2..9790e10ed 100644
--- a/main.go
+++ b/main.go
@@ -9,13 +9,13 @@ import (
"runtime"
"strconv"
- "github.com/trufflesecurity/trufflehog/pkg/common"
- "github.com/trufflesecurity/trufflehog/pkg/sources/git"
-
"github.com/sirupsen/logrus"
+ kingpin "gopkg.in/alecthomas/kingpin.v2"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
"github.com/trufflesecurity/trufflehog/pkg/decoders"
"github.com/trufflesecurity/trufflehog/pkg/engine"
- kingpin "gopkg.in/alecthomas/kingpin.v2"
+ "github.com/trufflesecurity/trufflehog/pkg/sources/git"
)
func main() {
@@ -39,23 +39,27 @@ func main() {
gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
githubScan := cli.Command("github", "Find credentials in GitHub repositories.")
- // githubScanTarget := githubScan.Arg("target", "GitHub target. Can be a repository, user or organization.").Required().String()
- // githubScanToken := githubScan.Flag("token", "GitHub token.").String()
+ githubScanEndpoint := githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+ githubScanRepos := githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+ githubScanOrgs := githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+ githubScanToken := githubScan.Flag("token", "GitHub token.").String()
gitlabScan := cli.Command("gitlab", "Find credentials in GitLab repositories.")
// gitlabScanTarget := gitlabScan.Arg("target", "GitLab target. Can be a repository, user or organization.").Required().String()
// gitlabScanToken := gitlabScan.Flag("token", "GitLab token.").String()
- // bitbucketScan := cli.Command("bitbucket", "Find credentials in Bitbucket repositories.")
+ bitbucketScan := cli.Command("bitbucket", "Find credentials in Bitbucket repositories.")
// bitbucketScanTarget := bitbucketScan.Arg("target", "Bitbucket target. Can be a repository, user or organization.").Required().String()
// bitbucketScanToken := bitbucketScan.Flag("token", "Bitbucket token.").String()
- // filesystemScan := cli.Command("filesystem", "Find credentials in filesystem.")
+ filesystemScan := cli.Command("filesystem", "Find credentials in a filesystem.")
// filesystemScanPath := filesystemScan.Arg("path", "Path to scan.").Required().String()
// filesystemScanRecursive := filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
// filesystemScanIncludePaths := filesystemScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
// filesystemScanExcludePaths := filesystemScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+ s3Scan := cli.Command("s3", "Find credentials in an S3 bucket.")
+
cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))
if *jsonOut {
@@ -93,9 +97,21 @@ func main() {
logrus.WithError(err).Fatal("Failed to scan git.")
}
case githubScan.FullCommand():
- log.Fatal("github not implemented")
+ if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
+ log.Fatal("You must specify at least one organization or repository.")
+ }
+ err = e.ScanGitHub(ctx, *githubScanEndpoint, *githubScanRepos, *githubScanOrgs, *githubScanToken, filter, *concurrency)
+ if err != nil {
+ logrus.WithError(err).Fatal("Failed to scan git.")
+ }
case gitlabScan.FullCommand():
log.Fatal("gitlab not implemented")
+ case bitbucketScan.FullCommand():
+ log.Fatal("bitbucket not implemented")
+ case filesystemScan.FullCommand():
+ log.Fatal("filesystem not implemented")
+ case s3Scan.FullCommand():
+ log.Fatal("s3 not implemented")
}
// deal with the results from e.ResultsChan()
diff --git a/pkg/engine/github.go b/pkg/engine/github.go
new file mode 100644
index 000000000..3441f9de8
--- /dev/null
+++ b/pkg/engine/github.go
@@ -0,0 +1,49 @@
+package engine
+
+import (
+ "context"
+
+ "github.com/sirupsen/logrus"
+ "google.golang.org/protobuf/proto"
+ anypb "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/trufflesecurity/trufflehog/pkg/common"
+ "github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/pkg/sources/github"
+)
+
+func (e *Engine) ScanGitHub(ctx context.Context, endpoint string, repos, orgs []string, token string, filter *common.Filter, concurrency int) error {
+ source := github.Source{}
+ connection := sourcespb.GitHub{
+ Endpoint: endpoint,
+ Organizations: orgs,
+ Repositories: repos,
+ }
+ if len(token) > 0 {
+ connection.Credential = &sourcespb.GitHub_Token{
+ Token: token,
+ }
+ } else {
+ connection.Credential = &sourcespb.GitHub_Unauthenticated{}
+ }
+ var conn anypb.Any
+ err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
+ if err != nil {
+ logrus.WithError(err).Error("failed to marshal github connection")
+ return err
+ }
+ err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, concurrency)
+ if err != nil {
+ logrus.WithError(err).Error("failed to initialize github source")
+ return err
+ }
+
+ go func() {
+ err := source.Chunks(ctx, e.ChunksChan())
+ if err != nil {
+ logrus.WithError(err).Fatal("could not scan github")
+ }
+ close(e.ChunksChan())
+ }()
+ return nil
+}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index b4a9622a3..f6f8967ab 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -140,11 +140,25 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
case *sourcespb.GitHub_Unauthenticated:
apiClient := github.NewClient(s.httpClient)
if len(s.orgs) > 30 {
- log.Warn("You may experience rate limiting due with the unauthenticated GitHub api. Consider using the authenticated org user scan feature instead")
+ log.Warn("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.")
}
- for _, org := range s.orgs {
- org = strings.TrimSpace(org)
- if !strings.HasSuffix(org, ".git") {
+
+ if len(s.repos) > 0 {
+ for i, repo := range s.repos {
+ if !strings.HasSuffix(repo, ".git") {
+ if repo, err := giturl.NormalizeGithubRepo(repo); err != nil {
+ // This wasn't formatted as expected, let the user know why that might be.
+ log.WithError(err).Warnf("Repo not in expected format, attempting to paginate repos instead.")
+ } else {
+ s.repos[i] = repo
+ }
+ s.paginateRepos(ctx, apiClient, repo)
+ }
+ }
+ }
+
+ if len(s.orgs) > 0 {
+ for _, org := range s.orgs {
s.paginateRepos(ctx, apiClient, org)
}
}
@@ -203,6 +217,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
if err != nil {
return errors.New(err)
}
+ // TODO: this should enumerate an organizations gists too...
s.paginateGists(ctx, user.GetLogin(), chunksChan)
if !specificScope {
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 3b0010abb..3751468d2 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -11,12 +11,13 @@ import (
"github.com/kylelemons/godebug/pretty"
"github.com/mattn/go-colorable"
log "github.com/sirupsen/logrus"
+ "google.golang.org/protobuf/types/known/anypb"
+
"github.com/trufflesecurity/trufflehog/pkg/common"
"github.com/trufflesecurity/trufflehog/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/pkg/sources"
- "google.golang.org/protobuf/types/known/anypb"
)
func TestSource_Scan(t *testing.T) {
@@ -192,6 +193,29 @@ func TestSource_Scan(t *testing.T) {
},
wantErr: false,
},
+ {
+ name: "get an unauthenticated repo chunk with no enumeration",
+ init: init{
+ name: "test source",
+ connection: &sourcespb.GitHub{
+ Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"},
+ Credential: &sourcespb.GitHub_Unauthenticated{},
+ },
+ },
+ wantChunk: &sources.Chunk{
+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ SourceName: "test source",
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Repository: "https://github.com/trufflesecurity/driftwood.git",
+ },
+ },
+ },
+ Verify: false,
+ },
+ wantErr: false,
+ },
{
name: "installed app on org w/ enum",
init: init{