Skip to content

Find, verify, and analyze leaked credentials

fix(github): use a single "app client registry" to make sure we reuse client connections in a process (#5427)

meredith27d8ea2876d9
diff --git a/go.mod b/go.mod
index 65a0248ac..19d57d6e4 100644
--- a/go.mod
+++ b/go.mod
@@ -72,6 +72,7 @@ require (
github.com/lrstanley/bubblezone v0.0.0-20250404061050-e13639e27357
github.com/marusama/semaphore/v2 v2.5.0
github.com/mattn/go-isatty v0.0.20
+ github.com/maypok86/otter/v2 v2.3.0
github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d
github.com/microsoft/go-mssqldb v1.8.2
github.com/mitchellh/go-ps v1.0.0
diff --git a/go.sum b/go.sum
index 757286f2c..4b9357435 100644
--- a/go.sum
+++ b/go.sum
@@ -543,6 +543,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei
github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w=
+github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8=
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d h1:Vw3f39TqFSQLA+OyW+8SouppHTYzX8/fDv6Ao8uj3Ho=
diff --git a/pkg/sources/github/app_client_registry.go b/pkg/sources/github/app_client_registry.go
new file mode 100644
index 000000000..9bc8c58aa
--- /dev/null
+++ b/pkg/sources/github/app_client_registry.go
@@ -0,0 +1,246 @@
+package github
+
+// This "registry" serves as a cache of github app clients across the whole
+// process, so we work with single instances of clients that don't need to
+// fetch access tokens separately and can better track rate limits.
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/url"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/bradleyfalzon/ghinstallation/v2"
+ "github.com/google/go-github/v67/github"
+ "github.com/maypok86/otter/v2"
+ "github.com/shurcooL/githubv4"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+var (
+ appClients *otter.Cache[appClientKey, *github.Client]
+ clientsByInstallationID *otter.Cache[appInstallationClientKey, *AppInstallationClients]
+)
+
+var initCaches = sync.OnceFunc(func() {
+ appClients = otter.Must(&otter.Options[appClientKey, *github.Client]{})
+
+ clientsByInstallationID = otter.Must(&otter.Options[appInstallationClientKey, *AppInstallationClients]{
+ MaximumSize: 1000,
+ })
+})
+
+type appClientKey struct {
+ apiEndpoint string
+ appID int64
+ privateKey string
+}
+
+type appInstallationClientKey struct {
+ appClientKey
+ installationID int64
+}
+
+type AppInstallationClients struct {
+ appClient *github.Client
+ InstallationID int64
+ APIClient *github.Client
+ GraphQLClient *githubv4.Client
+ tokenMu sync.RWMutex
+ token *github.InstallationToken
+}
+
+// AccessToken returns an installation access token. If not present yet or
+// expired, one is fetched. The expiry is adjusted so we refresh one minute
+// early.
+func (cs *AppInstallationClients) AccessToken(ctx context.Context) (string, error) {
+ if cs.InstallationID == 0 {
+ return "", fmt.Errorf("tried to fetch installation access token for id 0")
+ }
+
+ cs.tokenMu.RLock()
+ token := cs.accessTokenIfValid()
+ cs.tokenMu.RUnlock()
+
+ if token != nil {
+ return token.GetToken(), nil
+ }
+
+ cs.tokenMu.Lock()
+ defer cs.tokenMu.Unlock()
+
+ if token := cs.accessTokenIfValid(); token != nil {
+ return token.GetToken(), nil
+ }
+
+ token, _, err := cs.appClient.Apps.CreateInstallationToken(
+ ctx, cs.InstallationID, &github.InstallationTokenOptions{},
+ )
+ if err != nil {
+ return "", fmt.Errorf("could not create installation token for installation %d: %w", cs.InstallationID, err)
+ }
+
+ if exp := token.GetExpiresAt(); !exp.IsZero() {
+ // Refresh just before the real expiry
+ token.ExpiresAt = &github.Timestamp{Time: exp.Add(-time.Minute)}
+ }
+
+ cs.token = token
+
+ return token.GetToken(), nil
+}
+
+func (cs *AppInstallationClients) accessTokenIfValid() *github.InstallationToken {
+ if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) {
+ return nil
+ }
+ return cs.token
+}
+
+// GetAppClient creates or returns from cache a github API client for the given
+// app. This is the client that isn't associated with a specific installation.
+func GetAppClient(ctx context.Context, apiEndpoint string, appID int64, privateKey string) (*github.Client, error) {
+ initCaches()
+
+ loader := func(ctx context.Context, key appClientKey) (*github.Client, error) {
+ transport, err := newAppsTransport(key.apiEndpoint, key.appID, []byte(key.privateKey))
+ if err != nil {
+ return nil, fmt.Errorf("could not create app client transport: %w", err)
+ }
+
+ client, err := newGitHubClientWithTransport(key.apiEndpoint, transport)
+ if err != nil {
+ return nil, fmt.Errorf("could not create app client: %w", err)
+ }
+
+ return client, nil
+ }
+
+ return appClients.Get(
+ ctx,
+ appClientKey{apiEndpoint, appID, privateKey},
+ otter.LoaderFunc[appClientKey, *github.Client](loader),
+ )
+}
+
+// GetAppInstallationClients returns a struct holding API and GraphQL clients
+// for the requested app and installation, from cache if available.
+func GetAppInstallationClients(
+ ctx context.Context,
+ apiEndpoint string,
+ appID int64,
+ privateKey string,
+ instID int64,
+) (*AppInstallationClients, error) {
+ initCaches()
+
+ loader := func(ctx context.Context, key appInstallationClientKey) (*AppInstallationClients, error) {
+ ac, err := GetAppClient(ctx, key.apiEndpoint, key.appID, key.privateKey)
+ if err != nil {
+ return nil, err
+ }
+
+ appsTransport, err := newAppsTransport(key.apiEndpoint, key.appID, []byte(key.privateKey))
+ if err != nil {
+ return nil, fmt.Errorf("could not create app transport for installation %d: %w", instID, err)
+ }
+
+ // NewFromAppsTransport inherits BaseURL from appsTransport, which
+ // newAppsTransport already set correctly (incl. GHE.com). Don't override it
+ // with the raw endpoint here or GHE.com token refresh would 401.
+ transport := ghinstallation.NewFromAppsTransport(appsTransport, instID)
+
+ apiClient, err := newGitHubClientWithTransport(key.apiEndpoint, transport)
+ if err != nil {
+ return nil, fmt.Errorf("could not create API client for installation %d: %w", instID, err)
+ }
+
+ gqlClient, err := createGraphqlClient(trContext.AddLogger(ctx), apiClient.Client(), key.apiEndpoint)
+ if err != nil {
+ return nil, err
+ }
+
+ return &AppInstallationClients{
+ appClient: ac,
+ InstallationID: instID,
+ APIClient: apiClient,
+ GraphQLClient: gqlClient,
+ }, nil
+ }
+
+ return clientsByInstallationID.Get(
+ ctx,
+ appInstallationClientKey{appClientKey{apiEndpoint, appID, privateKey}, instID},
+ otter.LoaderFunc[appInstallationClientKey, *AppInstallationClients](loader),
+ )
+}
+
+func newAppsTransport(apiEndpoint string, appID int64, privateKey []byte) (*ghinstallation.AppsTransport, error) {
+ httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)
+ appsTransport, err := ghinstallation.NewAppsTransport(httpClient.Transport, appID, privateKey)
+ if err != nil {
+ return nil, err
+ }
+
+ // ghinstallation uses BaseURL to build the token exchange URL
+ // {BaseURL}/app/installations/{id}/access_tokens. For GHE.com this must be
+ // the api.SUBDOMAIN.ghe.com base with no /api/v3 and no trailing slash
+ // (which ghinstallation does not expect), otherwise token refresh 401s.
+ baseURL, err := appsBaseURL(apiEndpoint)
+ if err != nil {
+ return nil, err
+ }
+ appsTransport.BaseURL = strings.TrimRight(baseURL, "/")
+ return appsTransport, nil
+}
+
+// appsBaseURL returns the BaseURL that ghinstallation transports should use
+// for token exchange/refresh. For GHE.com it resolves to the api.* subdomain;
+// for what is likely GHES, it ensures /api/v3 is used; and for anything else
+// including github.com it is the endpoint as-is.
+func appsBaseURL(apiEndpoint string) (string, error) {
+ if isGHECloud(apiEndpoint) {
+ normalizedURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint)
+ if err != nil {
+ return "", fmt.Errorf("could not normalize GHE.com endpoint: %w", err)
+ }
+ return normalizedURL, nil
+ }
+
+ u, err := url.Parse(apiEndpoint)
+ if err != nil {
+ return "", fmt.Errorf("malformed endpoint url: %w", err)
+ }
+ h := u.Hostname()
+
+ if !strings.HasSuffix(u.Path, "/") {
+ u.Path += "/"
+ }
+
+ // If we're not dealing with github.com or GHE, then maybe GHES, which
+ // needs /api/v3 for ghinstallation find the right place
+ if !strings.HasPrefix(h, "api.") && !strings.Contains(h, ".api.") {
+ if !strings.HasSuffix(u.Path, "/api/v3/") {
+ u.Path += "api/v3/"
+ }
+ }
+
+ return u.String(), nil
+}
+
+func newGitHubClientWithTransport(apiEndpoint string, transport http.RoundTripper) (*github.Client, error) {
+ httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)
+ httpClient.Transport = transport
+
+ // GHE.com (GHEC with data residency) serves its REST API at the root of
+ // api.SUBDOMAIN.ghe.com (like api.github.com), NOT under the GHES /api/v3 path.
+ if isGHECloud(apiEndpoint) {
+ return createGHECloudClient(httpClient, apiEndpoint)
+ }
+ return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
+}
diff --git a/pkg/sources/github/app_client_registry_test.go b/pkg/sources/github/app_client_registry_test.go
new file mode 100644
index 000000000..295c655ec
--- /dev/null
+++ b/pkg/sources/github/app_client_registry_test.go
@@ -0,0 +1,154 @@
+package github
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/google/go-github/v67/github"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAccessTokenForInstallation(t *testing.T) {
+ privKey := generateTestPrivateKey(t)
+
+ accessTokenServer := func(accessTokenHandler func(w http.ResponseWriter, r *http.Request)) *httptest.Server {
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+
+ if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") {
+ accessTokenHandler(w, r)
+ return
+ }
+
+ // Default: return empty JSON array for any list endpoint.
+ _ = json.NewEncoder(w).Encode([]interface{}{})
+ }))
+ }
+
+ t.Run("invalid", func(t *testing.T) {
+ cs := &AppInstallationClients{InstallationID: 0}
+ _, err := cs.AccessToken(t.Context())
+ require.Error(t, err)
+ })
+
+ t.Run("success", func(t *testing.T) {
+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "token": "test-token",
+ "expires_at": "2099-01-01T00:00:00Z",
+ })
+ })
+ defer server.Close()
+
+ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)
+ require.NoError(t, err)
+
+ _, err = cs.AccessToken(t.Context())
+ require.NoError(t, err)
+ })
+
+ t.Run("rate limit", func(t *testing.T) {
+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("X-RateLimit-Remaining", "0")
+ w.WriteHeader(http.StatusForbidden)
+
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{})
+ })
+ defer server.Close()
+
+ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)
+ require.NoError(t, err)
+
+ _, err = cs.AccessToken(t.Context())
+ require.Error(t, err)
+
+ var gherr *github.RateLimitError
+ require.ErrorAs(t, err, &gherr)
+ })
+
+ t.Run("not found / no installation", func(t *testing.T) {
+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusNotFound)
+
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{})
+ })
+ defer server.Close()
+
+ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)
+ require.NoError(t, err)
+
+ _, err = cs.AccessToken(t.Context())
+ require.Error(t, err)
+ })
+
+ t.Run("reuse", func(t *testing.T) {
+ var hits atomic.Int64
+
+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "token": "test-token",
+ "expires_at": "2099-01-01T00:00:00Z",
+ })
+ })
+ defer server.Close()
+
+ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)
+ require.NoError(t, err)
+
+ _, err = cs.AccessToken(t.Context())
+ require.NoError(t, err)
+
+ _, err = cs.AccessToken(t.Context())
+ require.NoError(t, err)
+
+ require.Equal(t, int64(1), hits.Load())
+ })
+
+ t.Run("expire", func(t *testing.T) {
+ var hits atomic.Int64
+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "token": "test-token",
+ "expires_at": "2099-01-01T00:00:00Z",
+ })
+ })
+ defer server.Close()
+
+ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)
+ require.NoError(t, err)
+
+ _, err = cs.AccessToken(t.Context())
+ require.NoError(t, err)
+
+ token := cs.accessTokenIfValid()
+ require.NotNil(t, token)
+
+ token.ExpiresAt = &github.Timestamp{Time: time.Now()}
+
+ token = cs.accessTokenIfValid()
+ require.Nil(t, token)
+
+ // race these for good measure
+ var wg sync.WaitGroup
+ for range 3 {
+ wg.Go(func() {
+ _, err := cs.AccessToken(t.Context())
+ require.NoError(t, err)
+ })
+ }
+ wg.Wait()
+
+ token = cs.accessTokenIfValid()
+ require.NotNil(t, token)
+
+ require.Equal(t, int64(2), hits.Load())
+ })
+}
diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go
index 0739d4c9a..027d967ae 100644
--- a/pkg/sources/github/connector_app.go
+++ b/pkg/sources/github/connector_app.go
@@ -2,19 +2,13 @@ package github
import (
"fmt"
- "net/http"
- "net/url"
"strconv"
- "strings"
"sync"
- "time"
- "github.com/bradleyfalzon/ghinstallation/v2"
gogit "github.com/go-git/go-git/v5"
"github.com/google/go-github/v67/github"
"github.com/shurcooL/githubv4"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -24,13 +18,10 @@ type appConnector struct {
installationClient *github.Client
installationID int64
appID int64
- appPrivateKey []byte
apiEndpoint string
+ appPrivateKey string
mu sync.RWMutex
- // clientsByInstallationID is scoped to this connector/source run and is
- // bounded by the installations touched while scanning.
- clientsByInstallationID map[int64]*appInstallationClients
// repoInstallationMap maps repo clone URLs to their owning installation
// ID for repos discovered from non-default installations. Clone checks
@@ -38,20 +29,6 @@ type appConnector struct {
repoInstallationMap map[string]int64
}
-type appInstallationClients struct {
- apiClient *github.Client
- graphqlClient *githubv4.Client
- tokenMu sync.RWMutex
- token *github.InstallationToken
-}
-
-func (cs *appInstallationClients) accessTokenIfValid() *github.InstallationToken {
- if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) {
- return nil
- }
- return cs.token
-}
-
var _ Connector = (*appConnector)(nil)
const githubHTTPTimeoutSeconds = 60
@@ -76,23 +53,18 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllIn
return nil, fmt.Errorf("could not parse app ID %q: %w", app.AppId, err)
}
- installationTransport, err := newAppsTransport(apiEndpoint, appID, []byte(app.PrivateKey))
- if err != nil {
- return nil, fmt.Errorf("could not create installation client transport: %w", err)
- }
- installationClient, err := newGitHubClientWithTransport(apiEndpoint, installationTransport)
+ installationClient, err := GetAppClient(context.Background(), apiEndpoint, appID, app.PrivateKey)
if err != nil {
return nil, fmt.Errorf("could not create installation client: %w", err)
}
connector := &appConnector{
- installationClient: installationClient,
- installationID: installationID,
- appID: appID,
- appPrivateKey: []byte(app.PrivateKey),
- apiEndpoint: apiEndpoint,
- clientsByInstallationID: make(map[int64]*appInstallationClients),
- repoInstallationMap: make(map[string]int64),
+ installationClient: installationClient,
+ installationID: installationID,
+ appID: appID,
+ apiEndpoint: apiEndpoint,
+ appPrivateKey: app.PrivateKey,
+ repoInstallationMap: make(map[string]int64),
}
// When no default installation is configured (installationID == 0, only
@@ -149,7 +121,7 @@ func (c *appConnector) graphqlClientForInstallation(installID int64) (*githubv4.
return nil, err
}
- return clients.graphqlClient, nil
+ return clients.GraphQLClient, nil
}
func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) {
@@ -164,57 +136,17 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string
return "", nil, fmt.Errorf("no GitHub App installation resolved for repo %q; set githubApp.installationId to a fallback installation to scan repos outside installation listings (e.g. member repos with scanUsers) together with scanAllInstallations", repoURL)
}
- token, err := c.accessTokenForInstallation(ctx, installID)
- if err != nil {
- return "", nil, err
- }
-
- return git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...)
-}
-
-// accessTokenForInstallation returns an installation access token.
-func (c *appConnector) accessTokenForInstallation(ctx context.Context, installID int64) (string, error) {
- if installID == 0 {
- return "", fmt.Errorf("tried to fetch installation access token for id 0")
- }
-
- clients, err := c.clientsForInstallation(installID)
+ cs, err := c.clientsForInstallation(installID)
if err != nil {
- return "", err
- }
-
- clients.tokenMu.RLock()
- token := clients.accessTokenIfValid()
- clients.tokenMu.RUnlock()
-
- if token != nil {
- return token.GetToken(), nil
- }
-
- clients.tokenMu.Lock()
- defer clients.tokenMu.Unlock()
-
- if token := clients.accessTokenIfValid(); token != nil {
- return token.GetToken(), nil
+ return "", nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)
}
- token, _, err = c.installationClient.Apps.CreateInstallationToken(
- ctx,
- installID,
- &github.InstallationTokenOptions{},
- )
+ token, err := cs.AccessToken(ctx)
if err != nil {
- return "", fmt.Errorf("could not create installation token for installation %d: %w", installID, err)
- }
-
- if exp := token.GetExpiresAt(); !exp.IsZero() {
- // Refresh just before the real expiry
- token.ExpiresAt = &github.Timestamp{Time: exp.Add(-time.Minute)}
+ return "", nil, fmt.Errorf("could not fetch installation access token for installation %d: %w", installID, err)
}
- clients.token = token
-
- return token.GetToken(), nil
+ return git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...)
}
// installationIDForRepo returns the mapped installation ID for repoURL. When no
@@ -309,135 +241,9 @@ func (c *appConnector) APIClientForInstallation(installationID int64) (*github.C
return nil, err
}
- return clients.apiClient, nil
-}
-
-func (c *appConnector) clientsForInstallation(installationID int64) (*appInstallationClients, error) {
- c.mu.RLock()
- clients, ok := c.clientsByInstallationID[installationID]
- c.mu.RUnlock()
- if ok {
- return clients, nil
- }
-
- clients, err := c.clientsForInstallationLocked(installationID)
- if err != nil {
- return nil, err
- }
-
- return clients, nil
+ return clients.APIClient, nil
}
-func (c *appConnector) clientsForInstallationLocked(installationID int64) (*appInstallationClients, error) {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- if clients, ok := c.clientsByInstallationID[installationID]; ok {
- return clients, nil
- }
-
- clients, err := c.createAPIClientsForInstallation(installationID)
- if err != nil {
- return nil, err
- }
-
- c.clientsByInstallationID[installationID] = clients
-
- return clients, nil
-}
-
-// createAPIClientsForInstallation creates both types of clients at once and
-// saves along with the transport in an appInstallationClients, so those
-// structs are never in a partially-filled state.
-func (c *appConnector) createAPIClientsForInstallation(installationID int64) (*appInstallationClients, error) {
- appsTransport, err := newAppsTransport(c.apiEndpoint, c.appID, c.appPrivateKey)
- if err != nil {
- return nil, fmt.Errorf("could not create app transport for installation %d: %w", installationID, err)
- }
-
- // NewFromAppsTransport inherits BaseURL from appsTransport, which
- // newAppsTransport already set correctly (incl. GHE.com). Don't override it
- // with the raw endpoint here or GHE.com token refresh would 401.
- transport := ghinstallation.NewFromAppsTransport(appsTransport, installationID)
-
- apiClient, err := newGitHubClientWithTransport(c.apiEndpoint, transport)
- if err != nil {
- return nil, fmt.Errorf("could not create API client for installation %d: %w", installationID, err)
- }
-
- // the only thing this uses context for is a level 2 log line...
- gqlClient, err := createGraphqlClient(context.Background(), apiClient.Client(), c.apiEndpoint)
- if err != nil {
- return nil, err
- }
-
- return &appInstallationClients{
- apiClient: apiClient,
- graphqlClient: gqlClient,
- }, nil
-}
-
-func newAppsTransport(apiEndpoint string, appID int64, privateKey []byte) (*ghinstallation.AppsTransport, error) {
- httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)
- appsTransport, err := ghinstallation.NewAppsTransport(httpClient.Transport, appID, privateKey)
- if err != nil {
- return nil, err
- }
-
- // ghinstallation uses BaseURL to build the token exchange URL
- // {BaseURL}/app/installations/{id}/access_tokens. For GHE.com this must be
- // the api.SUBDOMAIN.ghe.com base with no /api/v3 and no trailing slash
- // (which ghinstallation does not expect), otherwise token refresh 401s.
- baseURL, err := appsBaseURL(apiEndpoint)
- if err != nil {
- return nil, err
- }
- appsTransport.BaseURL = strings.TrimRight(baseURL, "/")
- return appsTransport, nil
-}
-
-// appsBaseURL returns the BaseURL that ghinstallation transports should use
-// for token exchange/refresh. For GHE.com it resolves to the api.* subdomain;
-// for what is likely GHES, it ensures /api/v3 is used; and for anything else
-// including github.com it is the endpoint as-is.
-func appsBaseURL(apiEndpoint string) (string, error) {
- if isGHECloud(apiEndpoint) {
- normalizedURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint)
- if err != nil {
- return "", fmt.Errorf("could not normalize GHE.com endpoint: %w", err)
- }
- return normalizedURL, nil
- }
-
- u, err := url.Parse(apiEndpoint)
- if err != nil {
- return "", fmt.Errorf("malformed endpoint url: %w", err)
- }
- h := u.Hostname()
-
- if !strings.HasSuffix(u.Path, "/") {
- u.Path += "/"
- }
-
- // If we're not dealing with github.com or GHE, then maybe GHES, which
- // needs /api/v3 for ghinstallation find the right place
- if !strings.HasPrefix(h, "api.") && !strings.Contains(h, ".api.") {
- if !strings.HasSuffix(u.Path, "/api/v3/") {
- u.Path += "api/v3/"
- }
- }
-
- return u.String(), nil
-}
-
-func newGitHubClientWithTransport(apiEndpoint string, transport http.RoundTripper) (*github.Client, error) {
- httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)
- httpClient.Transport = transport
-
- // GHE.com (GHEC with data residency) serves its REST API at the root of
- // api.SUBDOMAIN.ghe.com (like api.github.com), NOT under the GHES /api/v3 path.
- if isGHECloud(apiEndpoint) {
- return createGHECloudClient(httpClient, apiEndpoint)
- }
- return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)
+func (c *appConnector) clientsForInstallation(installationID int64) (*AppInstallationClients, error) {
+ return GetAppInstallationClients(context.Background(), c.apiEndpoint, c.appID, c.appPrivateKey, installationID)
}
diff --git a/pkg/sources/github/connector_app_test.go b/pkg/sources/github/connector_app_test.go
index 2d5898395..a0ac2fe6c 100644
--- a/pkg/sources/github/connector_app_test.go
+++ b/pkg/sources/github/connector_app_test.go
@@ -10,9 +10,7 @@ import (
"net/http/httptest"
"strings"
"sync"
- "sync/atomic"
"testing"
- "time"
"github.com/google/go-github/v67/github"
"github.com/shurcooL/githubv4"
@@ -58,19 +56,24 @@ func TestAPIClientForInstallation(t *testing.T) {
}))
defer server.Close()
- connector := &appConnector{
- appID: 12345,
- appPrivateKey: privKey,
- apiEndpoint: server.URL,
- clientsByInstallationID: make(map[int64]*appInstallationClients),
- }
+ connector, err := NewAppConnector(
+ server.URL,
+ &credentialspb.GitHubApp{
+ PrivateKey: string(privKey),
+ AppId: "12345",
+ InstallationId: "42",
+ },
+ false,
+ )
+ require.NoError(t, err)
+ ac := connector.(*appConnector)
t.Run("creates distinct clients for different installations", func(t *testing.T) {
- client1, err := connector.APIClientForInstallation(111)
+ client1, err := ac.APIClientForInstallation(111)
require.NoError(t, err)
assert.NotNil(t, client1)
- client2, err := connector.APIClientForInstallation(222)
+ client2, err := ac.APIClientForInstallation(222)
require.NoError(t, err)
assert.NotNil(t, client2)
@@ -78,9 +81,9 @@ func TestAPIClientForInstallation(t *testing.T) {
})
t.Run("reuses clients for the same installation", func(t *testing.T) {
- client1, err := connector.APIClientForInstallation(333)
+ client1, err := ac.APIClientForInstallation(333)
require.NoError(t, err)
- client2, err := connector.APIClientForInstallation(333)
+ client2, err := ac.APIClientForInstallation(333)
require.NoError(t, err)
assert.Same(t, client1, client2)
@@ -91,7 +94,7 @@ func TestAPIClientForInstallation(t *testing.T) {
tokenRequestPaths = nil
mu.Unlock()
- client, err := connector.APIClientForInstallation(42)
+ client, err := ac.APIClientForInstallation(42)
require.NoError(t, err)
ctx := trContext.Background()
@@ -110,11 +113,11 @@ func TestAPIClientForInstallation(t *testing.T) {
ctx := trContext.Background()
- client1, err := connector.APIClientForInstallation(100)
+ client1, err := ac.APIClientForInstallation(100)
require.NoError(t, err)
_, _, _ = client1.Organizations.ListMembers(ctx, "org-a", nil)
- client2, err := connector.APIClientForInstallation(200)
+ client2, err := ac.APIClientForInstallation(200)
require.NoError(t, err)
_, _, _ = client2.Organizations.ListMembers(ctx, "org-b", nil)
@@ -156,6 +159,7 @@ func TestNewAppConnectorDefaultAPIClientUsesConfiguredInstallation(t *testing.T)
AppId: "12345",
}, false)
require.NoError(t, err)
+
require.NotNil(t, connector.APIClient())
require.NotNil(t, connector.GraphQLClient())
@@ -235,9 +239,8 @@ func TestAPIClientAndGraphQLClientNonNilWithoutDefaultInstallation(t *testing.T)
// error instead of attempting to use installation ID 0.
func TestCloneErrorsWithoutResolvedInstallation(t *testing.T) {
connector := &appConnector{
- installationID: 0,
- clientsByInstallationID: make(map[int64]*appInstallationClients),
- repoInstallationMap: make(map[string]int64),
+ installationID: 0,
+ repoInstallationMap: make(map[string]int64),
}
_, _, err := connector.Clone(trContext.Background(), "https://github.com/some-member/personal-repo.git")
@@ -245,185 +248,6 @@ func TestCloneErrorsWithoutResolvedInstallation(t *testing.T) {
assert.Contains(t, err.Error(), "no GitHub App installation resolved")
}
-func TestAccessTokenForInstallation(t *testing.T) {
- privKey := generateTestPrivateKey(t)
-
- accessTokenServer := func(accessTokenHandler func(w http.ResponseWriter, r *http.Request)) *httptest.Server {
- return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-Type", "application/json")
-
- if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") {
- accessTokenHandler(w, r)
- return
- }
-
- // Default: return empty JSON array for any list endpoint.
- _ = json.NewEncoder(w).Encode([]interface{}{})
- }))
- }
-
- t.Run("invalid", func(t *testing.T) {
- _, err := (&appConnector{}).accessTokenForInstallation(trContext.Background(), 0)
- require.Error(t, err)
- })
-
- t.Run("success", func(t *testing.T) {
- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
- _ = json.NewEncoder(w).Encode(map[string]interface{}{
- "token": "test-token",
- "expires_at": "2099-01-01T00:00:00Z",
- })
- })
- defer server.Close()
-
- c, err := NewAppConnector(
- server.URL,
- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},
- false,
- )
- require.NoError(t, err)
- ac := c.(*appConnector)
-
- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)
- require.NoError(t, err)
- })
-
- t.Run("rate limit", func(t *testing.T) {
- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
- w.Header().Add("X-RateLimit-Remaining", "0")
- w.WriteHeader(http.StatusForbidden)
-
- _ = json.NewEncoder(w).Encode(map[string]interface{}{})
- })
- defer server.Close()
-
- c, err := NewAppConnector(
- server.URL,
- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},
- false,
- )
- require.NoError(t, err)
- ac := c.(*appConnector)
-
- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)
- require.Error(t, err)
-
- var gherr *github.RateLimitError
- require.ErrorAs(t, err, &gherr)
- })
-
- t.Run("not found / no installation", func(t *testing.T) {
- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusNotFound)
-
- _ = json.NewEncoder(w).Encode(map[string]interface{}{})
- })
- defer server.Close()
-
- c, err := NewAppConnector(
- server.URL,
- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},
- false,
- )
- require.NoError(t, err)
- ac := c.(*appConnector)
-
- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)
- require.Error(t, err)
- })
-
- t.Run("reuse", func(t *testing.T) {
- var hits atomic.Int64
-
- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
- hits.Add(1)
- _ = json.NewEncoder(w).Encode(map[string]interface{}{
- "token": "test-token",
- "expires_at": "2099-01-01T00:00:00Z",
- })
- })
- defer server.Close()
-
- c, err := NewAppConnector(
- server.URL,
- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},
- false,
- )
- require.NoError(t, err)
- ac := c.(*appConnector)
-
- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)
- require.NoError(t, err)
-
- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)
- require.NoError(t, err)
-
- require.Equal(t, int64(1), hits.Load())
- })
-
- t.Run("expire", func(t *testing.T) {
- var hits atomic.Int64
- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {
- hits.Add(1)
- _ = json.NewEncoder(w).Encode(map[string]interface{}{
- "token": "test-token",
- "expires_at": "2099-01-01T00:00:00Z",
- })
- })
- defer server.Close()
-
- c, err := NewAppConnector(
- server.URL,
- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},
- false,
- )
- require.NoError(t, err)
- ac := c.(*appConnector)
-
- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)
- require.NoError(t, err)
-
- cs, err := ac.clientsForInstallation(100)
- require.NoError(t, err)
- token := cs.accessTokenIfValid()
- require.NotNil(t, token)
-
- token.ExpiresAt = &github.Timestamp{Time: time.Now()}
-
- token = cs.accessTokenIfValid()
- require.Nil(t, token)
-
- // race these for good measure
- var wg sync.WaitGroup
- for range 3 {
- wg.Go(func() {
- _, err := ac.accessTokenForInstallation(trContext.Background(), 100)
- require.NoError(t, err)
- })
- }
- wg.Wait()
-
- token = cs.accessTokenIfValid()
- require.NotNil(t, token)
-
- require.Equal(t, int64(2), hits.Load())
- })
-}
-
-func TestAPIClientForInstallationUsesConfiguredClient(t *testing.T) {
- defaultClient := github.NewClient(nil)
- connector := &appConnector{
- installationID: 100,
- clientsByInstallationID: map[int64]*appInstallationClients{
- 100: {apiClient: defaultClient},
- },
- }
-
- got, err := connector.APIClientForInstallation(100)
- require.NoError(t, err)
- assert.Same(t, defaultClient, got)
-}
-
func TestCloneUsesRepoInstallationMap(t *testing.T) {
connector := &appConnector{
installationID: 100,
@@ -475,19 +299,23 @@ func TestAPIClientForRepoUsesRepoInstallationMap(t *testing.T) {
}))
defer server.Close()
- connector := &appConnector{
- installationID: 100,
- appID: 12345,
- appPrivateKey: privKey,
- apiEndpoint: server.URL,
- clientsByInstallationID: make(map[int64]*appInstallationClients),
- repoInstallationMap: make(map[string]int64),
- }
- connector.setRepoInstallation("https://github.com/other-org/repo.git", 999)
+ connector, err := NewAppConnector(
+ server.URL,
+ &credentialspb.GitHubApp{
+ PrivateKey: string(privKey),
+ AppId: "12345",
+ InstallationId: "100",
+ },
+ false,
+ )
+ require.NoError(t, err)
+ ac := connector.(*appConnector)
+
+ ac.setRepoInstallation("https://github.com/other-org/repo.git", 999)
- client, err := connector.APIClientForRepo("https://github.com/other-org/repo.git")
+ client, err := ac.APIClientForRepo("https://github.com/other-org/repo.git")
require.NoError(t, err)
- cachedClient, err := connector.APIClientForRepo("https://github.com/other-org/repo.git")
+ cachedClient, err := ac.APIClientForRepo("https://github.com/other-org/repo.git")
require.NoError(t, err)
assert.Same(t, client, cachedClient)
@@ -509,44 +337,56 @@ func TestGraphQLClientForRepoCachesClients(t *testing.T) {
defer server.Close()
defaultGraphQLClient := &githubv4.Client{}
- connector := &appConnector{
- installationID: 100,
- appID: 12345,
- appPrivateKey: privKey,
- apiEndpoint: server.URL,
- clientsByInstallationID: map[int64]*appInstallationClients{
- 100: {graphqlClient: defaultGraphQLClient},
+
+ connector, err := NewAppConnector(
+ server.URL,
+ &credentialspb.GitHubApp{
+ PrivateKey: string(privKey),
+ AppId: "12345",
+ InstallationId: "100",
},
- repoInstallationMap: make(map[string]int64),
- }
- connector.setRepoInstallation("https://github.com/other-org/repo.git", 999)
+ false,
+ )
+ require.NoError(t, err)
+ ac := connector.(*appConnector)
+
+ clientsByInstallationID.Set(
+ appInstallationClientKey{
+ appClientKey{server.URL, 12345, string(privKey)},
+ 100,
+ },
+ &AppInstallationClients{
+ InstallationID: 100,
+ GraphQLClient: defaultGraphQLClient,
+ },
+ )
+ ac.setRepoInstallation("https://github.com/other-org/repo.git", 999)
ctx := trContext.Background()
t.Run("returns default client for default installation repos", func(t *testing.T) {
- got, err := connector.GraphQLClientForRepo(ctx, "https://github.com/default-org/repo.git")
+ got, err := ac.GraphQLClientForRepo(ctx, "https://github.com/default-org/repo.git")
require.NoError(t, err)
assert.Same(t, defaultGraphQLClient, got)
})
t.Run("caches clients for non-default installations", func(t *testing.T) {
- client1, err := connector.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")
+ client1, err := ac.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")
require.NoError(t, err)
assert.NotSame(t, defaultGraphQLClient, client1)
- client2, err := connector.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")
+ client2, err := ac.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")
require.NoError(t, err)
assert.Same(t, client1, client2)
- apiClient, err := connector.APIClientForRepo("https://github.com/other-org/repo.git")
+ apiClient, err := ac.APIClientForRepo("https://github.com/other-org/repo.git")
require.NoError(t, err)
- connector.mu.RLock()
- cachedClients := connector.clientsByInstallationID[999]
- connector.mu.RUnlock()
+ cachedClients, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 999)
+ require.NoError(t, err)
require.NotNil(t, cachedClients)
- assert.Same(t, apiClient, cachedClients.apiClient)
- assert.Same(t, client1, cachedClients.graphqlClient)
+ assert.Same(t, apiClient, cachedClients.APIClient)
+ assert.Same(t, client1, cachedClients.GraphQLClient)
})
}
diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go
index 29911c078..dad98fee8 100644
--- a/pkg/sources/github/github_test.go
+++ b/pkg/sources/github/github_test.go
@@ -327,7 +327,8 @@ func TestAddMembersByApp(t *testing.T) {
InstallationId: "1337",
AppId: "4141",
},
- }})
+ },
+ })
err := s.addMembersByApp(context.Background(), s.connector.(*appConnector), noopReporter())
assert.Nil(t, err)
assert.Equal(t, 3, len(s.memberCache))
@@ -434,7 +435,8 @@ func TestValidate_ScanAllInstallationsWithoutDefaultInstallation(t *testing.T) {
PrivateKey: createPrivateKey(),
AppId: "4141",
},
- }})
+ },
+ })
errs := s.Validate(context.Background())
assert.Empty(t, errs)
@@ -2330,11 +2332,8 @@ func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) {
func TestScanAllInstallationsTargetMappingErrorIsTargetedScanError(t *testing.T) {
s := &Source{
- conn: &sourcespb.GitHub{ScanAllInstallations: true},
- connector: &appConnector{
- clientsByInstallationID: make(map[int64]*appInstallationClients),
- repoInstallationMap: make(map[string]int64),
- },
+ conn: &sourcespb.GitHub{ScanAllInstallations: true},
+ connector: &appConnector{},
}
ctx := context.Background()
chunksChan := make(chan *sources.Chunk)