Skip to content

Find, verify, and analyze leaked credentials

[GitHub] Add a GraphQL client to the connector (#3837)

Richard Gomezbb899f20dc5e
diff --git a/go.mod b/go.mod
index 14bee3ee4..26cfe2ee8 100644
--- a/go.mod
+++ b/go.mod
@@ -271,6 +271,8 @@ require (
github.com/sendgrid/rest v2.6.9+incompatible // indirect
github.com/shirou/gopsutil/v3 v3.23.12 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
+ github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 // indirect
+ github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/skeema/knownhosts v1.3.0 // indirect
github.com/sorairolake/lzip-go v0.3.5 // indirect
diff --git a/go.sum b/go.sum
index c108efa0c..db2b9b0b7 100644
--- a/go.sum
+++ b/go.sum
@@ -736,6 +736,10 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shuheiktgw/go-travis v0.3.1 h1:SAT16mi77ccqogOslnXxBXzXbpeyChaIYUwi2aJpVZY=
github.com/shuheiktgw/go-travis v0.3.1/go.mod h1:avnFFDqJDdRHwlF9tgqvYi3asQCm/HGL8aLxYiKa4Yg=
+github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M=
+github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0=
+github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466/go.mod h1:9dIRpgIY7hVhoqfe0/FcYp0bpInZaT7dc3BYOprrIUE=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go
index d9d0f2ee3..126657cf7 100644
--- a/pkg/sources/github/connector.go
+++ b/pkg/sources/github/connector.go
@@ -1,17 +1,92 @@
package github
import (
+ "fmt"
+ "net/http"
+ "net/url"
+ "strings"
+
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v67/github"
+ "github.com/shurcooL/githubv4"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
)
-const cloudEndpoint = "https://api.github.com"
+const (
+ cloudV3Endpoint = "https://api.github.com"
+ cloudGraphqlEndpoint = "https://api.github.com/graphql" // https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#the-graphql-endpoint
+)
// Connector abstracts over the authenticated ways to interact with GitHub: cloning and API operations.
type Connector interface {
// APIClient returns a configured GitHub client that can be used for GitHub API operations.
APIClient() *github.Client
+ // GraphQLClient returns a client that can be used for GraphQL operations.
+ GraphQLClient() *githubv4.Client
// Clone clones a repository using the configured authentication information.
Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error)
}
+
+func newConnector(ctx context.Context, source *Source) (Connector, error) {
+ apiEndpoint := source.conn.Endpoint
+ if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) {
+ apiEndpoint = cloudV3Endpoint
+ }
+
+ switch cred := source.conn.GetCredential().(type) {
+ case *sourcespb.GitHub_GithubApp:
+ log.RedactGlobally(cred.GithubApp.GetPrivateKey())
+ return NewAppConnector(ctx, apiEndpoint, cred.GithubApp)
+ case *sourcespb.GitHub_BasicAuth:
+ log.RedactGlobally(cred.BasicAuth.GetPassword())
+ return NewBasicAuthConnector(ctx, apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth)
+ case *sourcespb.GitHub_Token:
+ log.RedactGlobally(cred.Token)
+ return NewTokenConnector(ctx, apiEndpoint, cred.Token, source.conn.GetClonePath(), source.useAuthInUrl, func(c context.Context, err error) bool {
+ return source.handleRateLimit(c, err)
+ })
+ case *sourcespb.GitHub_Unauthenticated:
+ return NewUnauthenticatedConnector(ctx, apiEndpoint, source.conn.GetClonePath())
+ default:
+ return nil, fmt.Errorf("unknown connection type %T", source.conn.GetCredential())
+ }
+}
+
+func createAPIClient(ctx context.Context, httpClient *http.Client, apiEndpoint string) (*github.Client, error) {
+ ctx.Logger().V(2).Info("Creating API client", "url", apiEndpoint)
+
+ // If we're using public GitHub, make a regular client.
+ // Otherwise, make an enterprise client.
+ if strings.EqualFold(apiEndpoint, cloudV3Endpoint) {
+ return github.NewClient(httpClient), nil
+ }
+
+ return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
+}
+
+func createGraphqlClient(ctx context.Context, client *http.Client, apiEndpoint string) (*githubv4.Client, error) {
+ var graphqlEndpoint string
+ if apiEndpoint == cloudV3Endpoint {
+ graphqlEndpoint = cloudGraphqlEndpoint
+ } else {
+ // Use the root endpoint for the host.
+ // https://docs.github.com/en/[email protected]/graphql/guides/introduction-to-graphql
+ parsedURL, err := url.Parse(apiEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("error parsing URL: %w", err)
+ }
+
+ // GitHub Enterprise uses `/api/v3` for the base. (https://github.com/google/go-github/issues/958)
+ // Swap it, and anything before `/api`, with GraphQL.
+ before, _ := strings.CutSuffix(parsedURL.Path, "/api/v3")
+ parsedURL.Path = before + "/api/graphql"
+ graphqlEndpoint = parsedURL.String()
+ }
+
+ ctx.Logger().V(2).Info("Creating GraphQL client", "url", graphqlEndpoint)
+
+ return githubv4.NewEnterpriseClient(graphqlEndpoint, client), nil
+}
diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go
index 0a709e13e..cbe623e4a 100644
--- a/pkg/sources/github/connector_app.go
+++ b/pkg/sources/github/connector_app.go
@@ -7,6 +7,8 @@ import (
"github.com/bradleyfalzon/ghinstallation/v2"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v67/github"
+ "github.com/shurcooL/githubv4"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
@@ -15,13 +17,14 @@ import (
type appConnector struct {
apiClient *github.Client
+ graphqlClient *githubv4.Client
installationClient *github.Client
installationID int64
}
var _ Connector = (*appConnector)(nil)
-func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (Connector, error) {
+func NewAppConnector(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (Connector, error) {
installationID, err := strconv.ParseInt(app.InstallationId, 10, 64)
if err != nil {
return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err)
@@ -67,8 +70,14 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (Connecto
return nil, fmt.Errorf("could not create API client: %w", err)
}
+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)
+ }
+
return &appConnector{
apiClient: apiClient,
+ graphqlClient: graphqlClient,
installationClient: installationClient,
installationID: installationID,
}, nil
@@ -91,6 +100,10 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string
return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "", "x-access-token", true, args...)
}
+func (c *appConnector) GraphQLClient() *githubv4.Client {
+ return c.graphqlClient
+}
+
func (c *appConnector) InstallationClient() *github.Client {
return c.installationClient
}
diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go
index 279f3cdb4..94aae56ab 100644
--- a/pkg/sources/github/connector_basicauth.go
+++ b/pkg/sources/github/connector_basicauth.go
@@ -5,6 +5,8 @@ import (
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v67/github"
+ "github.com/shurcooL/githubv4"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
@@ -12,15 +14,16 @@ import (
)
type basicAuthConnector struct {
- apiClient *github.Client
- username string
- password string
- clonePath string
+ apiClient *github.Client
+ graphqlClient *githubv4.Client
+ username string
+ password string
+ clonePath string
}
var _ Connector = (*basicAuthConnector)(nil)
-func NewBasicAuthConnector(apiEndpoint, clonePath string, cred *credentialspb.BasicAuth) (Connector, error) {
+func NewBasicAuthConnector(ctx context.Context, apiEndpoint, clonePath string, cred *credentialspb.BasicAuth) (Connector, error) {
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
httpClient.Transport = &github.BasicAuthTransport{
@@ -28,16 +31,22 @@ func NewBasicAuthConnector(apiEndpoint, clonePath string, cred *credentialspb.Ba
Password: cred.Password,
}
- apiClient, err := createGitHubClient(httpClient, apiEndpoint)
+ apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)
+ }
+
return &basicAuthConnector{
- apiClient: apiClient,
- username: cred.Username,
- password: cred.Password,
- clonePath: clonePath,
+ apiClient: apiClient,
+ graphqlClient: graphqlClient,
+ username: cred.Username,
+ password: cred.Password,
+ clonePath: clonePath,
}, nil
}
@@ -48,3 +57,7 @@ func (c *basicAuthConnector) APIClient() *github.Client {
func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) {
return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.clonePath, c.username, true, args...)
}
+
+func (c *basicAuthConnector) GraphQLClient() *githubv4.Client {
+ return c.graphqlClient
+}
diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go
index e00b8eb42..9bba8c124 100644
--- a/pkg/sources/github/connector_token.go
+++ b/pkg/sources/github/connector_token.go
@@ -7,15 +7,19 @@ import (
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v67/github"
+ "github.com/shurcooL/githubv4"
+ "golang.org/x/oauth2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
- "golang.org/x/oauth2"
)
type tokenConnector struct {
- apiClient *github.Client
- token string
+ token string
+ apiClient *github.Client
+ graphqlClient *githubv4.Client
+
isGitHubEnterprise bool
handleRateLimit func(context.Context, error) bool
user string
@@ -26,7 +30,7 @@ type tokenConnector struct {
var _ Connector = (*tokenConnector)(nil)
-func NewTokenConnector(apiEndpoint, token, clonePath string, authInUrl bool, handleRateLimit func(context.Context, error) bool) (Connector, error) {
+func NewTokenConnector(ctx context.Context, apiEndpoint, token, clonePath string, authInUrl bool, handleRateLimit func(context.Context, error) bool) (Connector, error) {
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
@@ -35,15 +39,21 @@ func NewTokenConnector(apiEndpoint, token, clonePath string, authInUrl bool, han
Source: tokenSource,
}
- apiClient, err := createGitHubClient(httpClient, apiEndpoint)
+ apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)
+ }
+
return &tokenConnector{
apiClient: apiClient,
+ graphqlClient: graphqlClient,
token: token,
- isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint),
+ isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudV3Endpoint),
handleRateLimit: handleRateLimit,
authInUrl: authInUrl,
clonePath: clonePath,
@@ -62,6 +72,10 @@ func (c *tokenConnector) Clone(ctx context.Context, repoURL string, args ...stri
return git.CloneRepoUsingToken(ctx, c.token, repoURL, c.clonePath, c.user, c.authInUrl, args...)
}
+func (c *tokenConnector) GraphQLClient() *githubv4.Client {
+ return c.graphqlClient
+}
+
func (c *tokenConnector) IsGithubEnterprise() bool {
return c.isGitHubEnterprise
}
diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go
index 977862156..8f93baa83 100644
--- a/pkg/sources/github/connector_unauthenticated.go
+++ b/pkg/sources/github/connector_unauthenticated.go
@@ -5,6 +5,7 @@ import (
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v67/github"
+ "github.com/shurcooL/githubv4"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -12,22 +13,31 @@ import (
)
type unauthenticatedConnector struct {
- apiClient *github.Client
+ apiClient *github.Client
+ graphqlClient *githubv4.Client
+
clonePath string
}
var _ Connector = (*unauthenticatedConnector)(nil)
-func NewUnauthenticatedConnector(apiEndpoint, clonePath string) (Connector, error) {
+func NewUnauthenticatedConnector(ctx context.Context, apiEndpoint, clonePath string) (Connector, error) {
const httpTimeoutSeconds = 60
httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds))
- apiClient, err := createGitHubClient(httpClient, apiEndpoint)
+ apiClient, err := createAPIClient(ctx, httpClient, apiEndpoint)
if err != nil {
return nil, fmt.Errorf("could not create API client: %w", err)
}
+
+ graphqlClient, err := createGraphqlClient(ctx, httpClient, apiEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("error creating GraphQL client: %w", err)
+ }
+
return &unauthenticatedConnector{
- apiClient: apiClient,
- clonePath: clonePath,
+ apiClient: apiClient,
+ graphqlClient: graphqlClient,
+ clonePath: clonePath,
}, nil
}
@@ -38,3 +48,7 @@ func (c *unauthenticatedConnector) APIClient() *github.Client {
func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) {
return git.CloneRepoUsingUnauthenticated(ctx, repoURL, c.clonePath, args...)
}
+
+func (c *unauthenticatedConnector) GraphQLClient() *githubv4.Client {
+ return c.graphqlClient
+}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 29a43eb9b..9912bbcb4 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -19,7 +19,6 @@ import (
"github.com/gobwas/glob"
"github.com/google/go-github/v67/github"
- "github.com/trufflesecurity/trufflehog/v3/pkg/log"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -235,7 +234,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
// configuration uses the inverse logic of the `useAuthInUrl` flag.
s.useAuthInUrl = !s.conn.RemoveAuthInUrl
- connector, err := newConnector(s)
+ connector, err := newConnector(aCtx, s)
if err != nil {
return fmt.Errorf("could not create connector: %w", err)
}
@@ -631,16 +630,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, installationClient *githu
return nil
}
-func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) {
- // If we're using public GitHub, make a regular client.
- // Otherwise, make an enterprise client.
- if strings.EqualFold(apiEndpoint, cloudEndpoint) {
- return github.NewClient(httpClient), nil
- }
-
- return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
-}
-
func (s *Source) scan(ctx context.Context, reporter sources.ChunkReporter) error {
var scannedCount uint64 = 1
@@ -1655,28 +1644,3 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte
}
return s.scanRepo(ctx, repoURL, reporter)
}
-
-func newConnector(source *Source) (Connector, error) {
- apiEndpoint := source.conn.Endpoint
- if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) {
- apiEndpoint = cloudEndpoint
- }
-
- switch cred := source.conn.GetCredential().(type) {
- case *sourcespb.GitHub_GithubApp:
- log.RedactGlobally(cred.GithubApp.GetPrivateKey())
- return NewAppConnector(apiEndpoint, cred.GithubApp)
- case *sourcespb.GitHub_BasicAuth:
- log.RedactGlobally(cred.BasicAuth.GetPassword())
- return NewBasicAuthConnector(apiEndpoint, source.conn.GetClonePath(), cred.BasicAuth)
- case *sourcespb.GitHub_Token:
- log.RedactGlobally(cred.Token)
- return NewTokenConnector(apiEndpoint, cred.Token, source.conn.GetClonePath(), source.useAuthInUrl, func(c context.Context, err error) bool {
- return source.handleRateLimit(c, err)
- })
- case *sourcespb.GitHub_Unauthenticated:
- return NewUnauthenticatedConnector(apiEndpoint, source.conn.GetClonePath())
- default:
- return nil, fmt.Errorf("unknown connection type %T", source.conn.GetCredential())
- }
-}