Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[GitHub] Add a GraphQL client to the connector (#3837)
Richard Gomezbb899f20dc5e
diff --git a/go.mod b/go.modindex 14bee3ee4..26cfe2ee8 100644--- a/go.mod+++ b/go.mod@@ -271,6 +271,8 @@ require (github.com/sendgrid/rest v2.6.9+incompatible // indirectgithub.com/shirou/gopsutil/v3 v3.23.12 // indirectgithub.com/shoenig/go-m1cpu v0.1.6 // indirect+ github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 // indirect+ github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirectgithub.com/sirupsen/logrus v1.9.3 // indirectgithub.com/skeema/knownhosts v1.3.0 // indirectgithub.com/sorairolake/lzip-go v0.3.5 // indirectdiff --git a/go.sum b/go.sumindex c108efa0c..db2b9b0b7 100644--- a/go.sum+++ b/go.sum@@ -736,6 +736,10 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=+github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M=+github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8=+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0=+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE=github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.goindex d9d0f2ee3..126657cf7 100644--- a/pkg/sources/github/connector.go+++ b/pkg/sources/github/connector.go@@ -1,17 +1,92 @@package githubimport (+ "fmt"+ "net/http"+ "net/url"+ "strings"+gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v67/github"+ "github.com/shurcooL/githubv4"+"github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/log"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb")-const cloudEndpoint = "https://api.github.com"+const (+ cloudV3Endpoint = "https://api.github.com"+ cloudGraphqlEndpoint = "https://api.github.com/graphql" // https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#the-graphql-endpoint+)// Connector abstracts over the authenticated ways to interact with GitHub: cloning and API operations.type Connector interface {// APIClient returns a configured GitHub client that can be used for GitHub API operations.APIClient() *github.Client+ // GraphQLClient returns a client that can be used for GraphQL operations.+ GraphQLClient() *githubv4.Client// Clone clones a repository using the configured authentication information.Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error)}++func newConnector(ctx context.Context, source *Source) (Connector, error) {+ apiEndpoint := source.conn.Endpoint+ if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) {+ apiEndpoint = cloudV3Endpoint+ }++ switch cred := source.conn.GetCredential().(type) {+ case *sourcespb.GitHub_GithubApp:+ log.RedactGlobally(cred.GithubApp.GetPrivateKey())+ return NewAppConnector(ctx, apiEndpoint, cred.GithubApp)+ case *sourcespb.GitHub_BasicAuth:+ log.RedactGlobally(cred.BasicAuth.GetPassword())+ return NewBasicAuthConnector(ctx, apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth)+ case *sourcespb.GitHub_Token:+ log.RedactGlobally(cred.Token)+ return NewTokenConnector(ctx, apiEndpoint, cred.Token, source.conn.GetClonePath(), source.useAuthInUrl, func(c context.Context, err error) bool {+ return source.handleRateLimit(c, err)+ })+ case *sourcespb.GitHub_Unauthenticated:+ return NewUnauthenticatedConnector(ctx, apiEndpoint, source.conn.GetClonePath())+ default:+ return nil, fmt.Errorf("unknown connection type %T", source.conn.GetCredential())+ }+}++func createAPIClient(ctx context.Context, httpClient *http.Client, apiEndpoint string) (*github.Client, error) {+ ctx.Logger().V(2).Info("Creating API client", "url", apiEndpoint)++ // If we're using public GitHub, make a regular client.+ // Otherwise, make an enterprise client.+ if strings.EqualFold(apiEndpoint, cloudV3Endpoint) {+ return github.NewClient(httpClient), nil+ }++ return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)+}++func createGraphqlClient(ctx context.Context, client *http.Client, apiEndpoint string) (*githubv4.Client, error) {+ var graphqlEndpoint string+ if apiEndpoint == cloudV3Endpoint {+ graphqlEndpoint = cloudGraphqlEndpoint+ } else {+ // Use the root endpoint for the host.+ // https://docs.github.com/en/[email protected]/graphql/guides/introduction-to-graphql+ parsedURL, err := url.Parse(apiEndpoint)+ if err != nil {+ return nil, fmt.Errorf("error parsing URL: %w", err)+ }++ // GitHub Enterprise uses `/api/v3` for the base. (https://github.com/google/go-github/issues/958)+ // Swap it, and anything before `/api`, with GraphQL.+ before, _ := strings.CutSuffix(parsedURL.Path, "/api/v3")+ parsedURL.Path = before + "/api/graphql"+ graphqlEndpoint = parsedURL.String()+ }++ ctx.Logger().V(2).Info("Creating GraphQL client", "url", graphqlEndpoint)++ return githubv4.NewEnterpriseClient(graphqlEndpoint, client), nil+}diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.goindex 0a709e13e..cbe623e4a 100644--- a/pkg/sources/github/connector_app.go+++ b/pkg/sources/github/connector_app.go@@ -7,6 +7,8 @@ import ("github.com/bradleyfalzon/ghinstallation/v2"gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v67/github"+ "github.com/shurcooL/githubv4"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"@@ -15,13 +17,14 @@ import (type appConnector struct {apiClient *github.Client+ graphqlClient *githubv4.ClientinstallationClient *github.ClientinstallationID int64}var _ Connector = (*appConnector)(nil)-func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (Connector, error) {+func NewAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (Connector, error) {installationID, err := strconv.ParseInt(app.InstallationId, 10, 64)if err != nil {return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err)@@ -67,8 +70,14 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (Connectoreturn nil, fmt.Errorf("could not create API client: %w", err)}+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)+ if err != nil {+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)+ }+return &appConnector{apiClient: apiClient,+ graphqlClient: graphqlClient,installationClient: installationClient,installationID: installationID,}, nil@@ -91,6 +100,10 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...stringreturn git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "", "x-access-token", true, args...)}+func (c *appConnector) GraphQLClient() *githubv4.Client {+ return c.graphqlClient+}+func (c *appConnector) InstallationClient() *github.Client {return c.installationClient}diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.goindex 279f3cdb4..94aae56ab 100644--- a/pkg/sources/github/connector_basicauth.go+++ b/pkg/sources/github/connector_basicauth.go@@ -5,6 +5,8 @@ import (gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v67/github"+ "github.com/shurcooL/githubv4"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"@@ -12,15 +14,16 @@ import ()type basicAuthConnector struct {- apiClient *github.Client- username string- password string- clonePath string+ apiClient *github.Client+ graphqlClient *githubv4.Client+ username string+ password string+ clonePath string}var _ Connector = (*basicAuthConnector)(nil)-func NewBasicAuthConnector(apiEndpoint, clonePath string, cred *credentialspb.BasicAuth) (Connector, error) {+func NewBasicAuthConnector(ctx context.Context, apiEndpoint, clonePath string, cred *credentialspb.BasicAuth) (Connector, error) {const httpTimeoutSeconds = 60httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))httpClient.Transport = &github.BasicAuthTransport{@@ -28,16 +31,22 @@ func NewBasicAuthConnector(apiEndpoint, clonePath string, cred *credentialspb.BaPassword: cred.Password,}- apiClient, err := createGitHubClient(httpClient, apiEndpoint)+ apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)if err != nil {return nil, fmt.Errorf("could not create API client: %w", err)}+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)+ if err != nil {+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)+ }+return &basicAuthConnector{- apiClient: apiClient,- username: cred.Username,- password: cred.Password,- clonePath: clonePath,+ apiClient: apiClient,+ graphqlClient: graphqlClient,+ username: cred.Username,+ password: cred.Password,+ clonePath: clonePath,}, nil}@@ -48,3 +57,7 @@ func (c *basicAuthConnector) APIClient() *github.Client {func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) {return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.clonePath, c.username, true, args...)}++func (c *basicAuthConnector) GraphQLClient() *githubv4.Client {+ return c.graphqlClient+}diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.goindex e00b8eb42..9bba8c124 100644--- a/pkg/sources/github/connector_token.go+++ b/pkg/sources/github/connector_token.go@@ -7,15 +7,19 @@ import (gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v67/github"+ "github.com/shurcooL/githubv4"+ "golang.org/x/oauth2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"- "golang.org/x/oauth2")type tokenConnector struct {- apiClient *github.Client- token string+ token string+ apiClient *github.Client+ graphqlClient *githubv4.Client+isGitHubEnterprise boolhandleRateLimit func(context.Context, error) booluser string@@ -26,7 +30,7 @@ type tokenConnector struct {var _ Connector = (*tokenConnector)(nil)-func NewTokenConnector(apiEndpoint, token, clonePath string, authInUrl bool, handleRateLimit func(context.Context, error) bool) (Connector, error) {+func NewTokenConnector(ctx context.Context, apiEndpoint, token, clonePath string, authInUrl bool, handleRateLimit func(context.Context, error) bool) (Connector, error) {const httpTimeoutSeconds = 60httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})@@ -35,15 +39,21 @@ func NewTokenConnector(apiEndpoint, token, clonePath string, authInUrl bool, hanSource: tokenSource,}- apiClient, err := createGitHubClient(httpClient, apiEndpoint)+ apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)if err != nil {return nil, fmt.Errorf("could not create API client: %w", err)}+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)+ if err != nil {+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)+ }+return &tokenConnector{apiClient: apiClient,+ graphqlClient: graphqlClient,token: token,- isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint),+ isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudV3Endpoint),handleRateLimit: handleRateLimit,authInUrl: authInUrl,clonePath: clonePath,@@ -62,6 +72,10 @@ func (c *tokenConnector) Clone(ctx context.Context, repoURL string, args ...strireturn git.CloneRepoUsingToken(ctx, c.token, repoURL, c.clonePath, c.user, c.authInUrl, args...)}+func (c *tokenConnector) GraphQLClient() *githubv4.Client {+ return c.graphqlClient+}+func (c *tokenConnector) IsGithubEnterprise() bool {return c.isGitHubEnterprise}diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.goindex 977862156..8f93baa83 100644--- a/pkg/sources/github/connector_unauthenticated.go+++ b/pkg/sources/github/connector_unauthenticated.go@@ -5,6 +5,7 @@ import (gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v67/github"+ "github.com/shurcooL/githubv4""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -12,22 +13,31 @@ import ()type unauthenticatedConnector struct {- apiClient *github.Client+ apiClient *github.Client+ graphqlClient *githubv4.Client+clonePath string}var _ Connector = (*unauthenticatedConnector)(nil)-func NewUnauthenticatedConnector(apiEndpoint, clonePath string) (Connector, error) {+func NewUnauthenticatedConnector(ctx context.Context, apiEndpoint, clonePath string) (Connector, error) {const httpTimeoutSeconds = 60httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))- apiClient, err := createGitHubClient(httpClient, apiEndpoint)+ apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)if err != nil {return nil, fmt.Errorf("could not create API client: %w", err)}++ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)+ if err != nil {+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)+ }+return &unauthenticatedConnector{- apiClient: apiClient,- clonePath: clonePath,+ apiClient: apiClient,+ graphqlClient: graphqlClient,+ clonePath: clonePath,}, nil}@@ -38,3 +48,7 @@ func (c *unauthenticatedConnector) APIClient() *github.Client {func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) {return git.CloneRepoUsingUnauthenticated(ctx, repoURL, c.clonePath, args...)}++func (c *unauthenticatedConnector) GraphQLClient() *githubv4.Client {+ return c.graphqlClient+}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 29a43eb9b..9912bbcb4 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -19,7 +19,6 @@ import ("github.com/gobwas/glob""github.com/google/go-github/v67/github"- "github.com/trufflesecurity/trufflehog/v3/pkg/log""golang.org/x/sync/errgroup""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/anypb"@@ -235,7 +234,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so// configuration uses the inverse logic of the `useAuthInUrl` flag.s.useAuthInUrl = !s.conn.RemoveAuthInUrl- connector, err := newConnector(s)+ connector, err := newConnector(aCtx, s)if err != nil {return fmt.Errorf("could not create connector: %w", err)}@@ -631,16 +630,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, installationClient *githureturn nil}-func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) {- // If we're using public GitHub, make a regular client.- // Otherwise, make an enterprise client.- if strings.EqualFold(apiEndpoint, cloudEndpoint) {- return github.NewClient(httpClient), nil- }-- return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)-}-func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error {var scannedCount uint64 = 1@@ -1655,28 +1644,3 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte}return s.scanRepo(ctx, repoURL, reporter)}--func newConnector(source *Source) (Connector, error) {- apiEndpoint := source.conn.Endpoint- if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) {- apiEndpoint = cloudEndpoint- }-- switch cred := source.conn.GetCredential().(type) {- case *sourcespb.GitHub_GithubApp:- log.RedactGlobally(cred.GithubApp.GetPrivateKey())- return NewAppConnector(apiEndpoint, cred.GithubApp)- case *sourcespb.GitHub_BasicAuth:- log.RedactGlobally(cred.BasicAuth.GetPassword())- return NewBasicAuthConnector(apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth)- case *sourcespb.GitHub_Token:- log.RedactGlobally(cred.Token)- return NewTokenConnector(apiEndpoint, cred.Token, source.conn.GetClonePath(), source.useAuthInUrl, func(c context.Context, err error) bool {- return source.handleRateLimit(c, err)- })- case *sourcespb.GitHub_Unauthenticated:- return NewUnauthenticatedConnector(apiEndpoint, source.conn.GetClonePath())- default:- return nil, fmt.Errorf("unknown connection type %T", source.conn.GetCredential())- }-}