Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix(github): use a single "app client registry" to make sure we reuse client connections in a process (#5427)
meredith27d8ea2876d9
diff --git a/go.mod b/go.modindex 65a0248ac..19d57d6e4 100644--- a/go.mod+++ b/go.mod@@ -72,6 +72,7 @@ require (github.com/lrstanley/bubblezone v0.0.0-20250404061050-e13639e27357github.com/marusama/semaphore/v2 v2.5.0github.com/mattn/go-isatty v0.0.20+ github.com/maypok86/otter/v2 v2.3.0github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73dgithub.com/microsoft/go-mssqldb v1.8.2github.com/mitchellh/go-ps v1.0.0diff --git a/go.sum b/go.sumindex 757286f2c..4b9357435 100644--- a/go.sum+++ b/go.sum@@ -543,6 +543,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Eigithub.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=+github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w=+github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8=github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d h1:Vw3f39TqFSQLA+OyW+8SouppHTYzX8/fDv6Ao8uj3Ho=diff --git a/pkg/sources/github/app_client_registry.go b/pkg/sources/github/app_client_registry.gonew file mode 100644index 000000000..9bc8c58aa--- /dev/null+++ b/pkg/sources/github/app_client_registry.go@@ -0,0 +1,246 @@+package github++// This "registry" serves as a cache of github app clients across the whole+// process, so we work with single instances of clients that don't need to+// fetch access tokens separately and can better track rate limits.++import (+ "context"+ "fmt"+ "net/http"+ "net/url"+ "strings"+ "sync"+ "time"++ "github.com/bradleyfalzon/ghinstallation/v2"+ "github.com/google/go-github/v67/github"+ "github.com/maypok86/otter/v2"+ "github.com/shurcooL/githubv4"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++var (+ appClients *otter.Cache[appClientKey, *github.Client]+ clientsByInstallationID *otter.Cache[appInstallationClientKey, *AppInstallationClients]+)++var initCaches = sync.OnceFunc(func() {+ appClients = otter.Must(&otter.Options[appClientKey, *github.Client]{})++ clientsByInstallationID = otter.Must(&otter.Options[appInstallationClientKey, *AppInstallationClients]{+ MaximumSize: 1000,+ })+})++type appClientKey struct {+ apiEndpoint string+ appID int64+ privateKey string+}++type appInstallationClientKey struct {+ appClientKey+ installationID int64+}++type AppInstallationClients struct {+ appClient *github.Client+ InstallationID int64+ APIClient *github.Client+ GraphQLClient *githubv4.Client+ tokenMu sync.RWMutex+ token *github.InstallationToken+}++// AccessToken returns an installation access token. If not present yet or+// expired, one is fetched. The expiry is adjusted so we refresh one minute+// early.+func (cs *AppInstallationClients) AccessToken(ctx context.Context) (string, error) {+ if cs.InstallationID == 0 {+ return "", fmt.Errorf("tried to fetch installation access token for id 0")+ }++ cs.tokenMu.RLock()+ token := cs.accessTokenIfValid()+ cs.tokenMu.RUnlock()++ if token != nil {+ return token.GetToken(), nil+ }++ cs.tokenMu.Lock()+ defer cs.tokenMu.Unlock()++ if token := cs.accessTokenIfValid(); token != nil {+ return token.GetToken(), nil+ }++ token, _, err := cs.appClient.Apps.CreateInstallationToken(+ ctx, cs.InstallationID, &github.InstallationTokenOptions{},+ )+ if err != nil {+ return "", fmt.Errorf("could not create installation token for installation %d: %w", cs.InstallationID, err)+ }++ if exp := token.GetExpiresAt(); !exp.IsZero() {+ // Refresh just before the real expiry+ token.ExpiresAt = &github.Timestamp{Time: exp.Add(-time.Minute)}+ }++ cs.token = token++ return token.GetToken(), nil+}++func (cs *AppInstallationClients) accessTokenIfValid() *github.InstallationToken {+ if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) {+ return nil+ }+ return cs.token+}++// GetAppClient creates or returns from cache a github API client for the given+// app. This is the client that isn't associated with a specific installation.+func GetAppClient(ctx context.Context, apiEndpoint string, appID int64, privateKey string) (*github.Client, error) {+ initCaches()++ loader := func(ctx context.Context, key appClientKey) (*github.Client, error) {+ transport, err := newAppsTransport(key.apiEndpoint, key.appID, []byte(key.privateKey))+ if err != nil {+ return nil, fmt.Errorf("could not create app client transport: %w", err)+ }++ client, err := newGitHubClientWithTransport(key.apiEndpoint, transport)+ if err != nil {+ return nil, fmt.Errorf("could not create app client: %w", err)+ }++ return client, nil+ }++ return appClients.Get(+ ctx,+ appClientKey{apiEndpoint, appID, privateKey},+ otter.LoaderFunc[appClientKey, *github.Client](loader),+ )+}++// GetAppInstallationClients returns a struct holding API and GraphQL clients+// for the requested app and installation, from cache if available.+func GetAppInstallationClients(+ ctx context.Context,+ apiEndpoint string,+ appID int64,+ privateKey string,+ instID int64,+) (*AppInstallationClients, error) {+ initCaches()++ loader := func(ctx context.Context, key appInstallationClientKey) (*AppInstallationClients, error) {+ ac, err := GetAppClient(ctx, key.apiEndpoint, key.appID, key.privateKey)+ if err != nil {+ return nil, err+ }++ appsTransport, err := newAppsTransport(key.apiEndpoint, key.appID, []byte(key.privateKey))+ if err != nil {+ return nil, fmt.Errorf("could not create app transport for installation %d: %w", instID, err)+ }++ // NewFromAppsTransport inherits BaseURL from appsTransport, which+ // newAppsTransport already set correctly (incl. GHE.com). Don't override it+ // with the raw endpoint here or GHE.com token refresh would 401.+ transport := ghinstallation.NewFromAppsTransport(appsTransport, instID)++ apiClient, err := newGitHubClientWithTransport(key.apiEndpoint, transport)+ if err != nil {+ return nil, fmt.Errorf("could not create API client for installation %d: %w", instID, err)+ }++ gqlClient, err := createGraphqlClient(trContext.AddLogger(ctx), apiClient.Client(), key.apiEndpoint)+ if err != nil {+ return nil, err+ }++ return &AppInstallationClients{+ appClient: ac,+ InstallationID: instID,+ APIClient: apiClient,+ GraphQLClient: gqlClient,+ }, nil+ }++ return clientsByInstallationID.Get(+ ctx,+ appInstallationClientKey{appClientKey{apiEndpoint, appID, privateKey}, instID},+ otter.LoaderFunc[appInstallationClientKey, *AppInstallationClients](loader),+ )+}++func newAppsTransport(apiEndpoint string, appID int64, privateKey []byte) (*ghinstallation.AppsTransport, error) {+ httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)+ appsTransport, err := ghinstallation.NewAppsTransport(httpClient.Transport, appID, privateKey)+ if err != nil {+ return nil, err+ }++ // ghinstallation uses BaseURL to build the token exchange URL+ // {BaseURL}/app/installations/{id}/access_tokens. For GHE.com this must be+ // the api.SUBDOMAIN.ghe.com base with no /api/v3 and no trailing slash+ // (which ghinstallation does not expect), otherwise token refresh 401s.+ baseURL, err := appsBaseURL(apiEndpoint)+ if err != nil {+ return nil, err+ }+ appsTransport.BaseURL = strings.TrimRight(baseURL, "/")+ return appsTransport, nil+}++// appsBaseURL returns the BaseURL that ghinstallation transports should use+// for token exchange/refresh. For GHE.com it resolves to the api.* subdomain;+// for what is likely GHES, it ensures /api/v3 is used; and for anything else+// including github.com it is the endpoint as-is.+func appsBaseURL(apiEndpoint string) (string, error) {+ if isGHECloud(apiEndpoint) {+ normalizedURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint)+ if err != nil {+ return "", fmt.Errorf("could not normalize GHE.com endpoint: %w", err)+ }+ return normalizedURL, nil+ }++ u, err := url.Parse(apiEndpoint)+ if err != nil {+ return "", fmt.Errorf("malformed endpoint url: %w", err)+ }+ h := u.Hostname()++ if !strings.HasSuffix(u.Path, "/") {+ u.Path += "/"+ }++ // If we're not dealing with github.com or GHE, then maybe GHES, which+ // needs /api/v3 for ghinstallation find the right place+ if !strings.HasPrefix(h, "api.") && !strings.Contains(h, ".api.") {+ if !strings.HasSuffix(u.Path, "/api/v3/") {+ u.Path += "api/v3/"+ }+ }++ return u.String(), nil+}++func newGitHubClientWithTransport(apiEndpoint string, transport http.RoundTripper) (*github.Client, error) {+ httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)+ httpClient.Transport = transport++ // GHE.com (GHEC with data residency) serves its REST API at the root of+ // api.SUBDOMAIN.ghe.com (like api.github.com), NOT under the GHES /api/v3 path.+ if isGHECloud(apiEndpoint) {+ return createGHECloudClient(httpClient, apiEndpoint)+ }+ return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)+}diff --git a/pkg/sources/github/app_client_registry_test.go b/pkg/sources/github/app_client_registry_test.gonew file mode 100644index 000000000..295c655ec--- /dev/null+++ b/pkg/sources/github/app_client_registry_test.go@@ -0,0 +1,154 @@+package github++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/google/go-github/v67/github"+ "github.com/stretchr/testify/require"+)++func TestAccessTokenForInstallation(t *testing.T) {+ privKey := generateTestPrivateKey(t)++ accessTokenServer := func(accessTokenHandler func(w http.ResponseWriter, r *http.Request)) *httptest.Server {+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")++ if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") {+ accessTokenHandler(w, r)+ return+ }++ // Default: return empty JSON array for any list endpoint.+ _ = json.NewEncoder(w).Encode([]interface{}{})+ }))+ }++ t.Run("invalid", func(t *testing.T) {+ cs := &AppInstallationClients{InstallationID: 0}+ _, err := cs.AccessToken(t.Context())+ require.Error(t, err)+ })++ t.Run("success", func(t *testing.T) {+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "token": "test-token",+ "expires_at": "2099-01-01T00:00:00Z",+ })+ })+ defer server.Close()++ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)+ require.NoError(t, err)++ _, err = cs.AccessToken(t.Context())+ require.NoError(t, err)+ })++ t.Run("rate limit", func(t *testing.T) {+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Add("X-RateLimit-Remaining", "0")+ w.WriteHeader(http.StatusForbidden)++ _ = json.NewEncoder(w).Encode(map[string]interface{}{})+ })+ defer server.Close()++ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)+ require.NoError(t, err)++ _, err = cs.AccessToken(t.Context())+ require.Error(t, err)++ var gherr *github.RateLimitError+ require.ErrorAs(t, err, &gherr)+ })++ t.Run("not found / no installation", func(t *testing.T) {+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusNotFound)++ _ = json.NewEncoder(w).Encode(map[string]interface{}{})+ })+ defer server.Close()++ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)+ require.NoError(t, err)++ _, err = cs.AccessToken(t.Context())+ require.Error(t, err)+ })++ t.Run("reuse", func(t *testing.T) {+ var hits atomic.Int64++ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "token": "test-token",+ "expires_at": "2099-01-01T00:00:00Z",+ })+ })+ defer server.Close()++ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)+ require.NoError(t, err)++ _, err = cs.AccessToken(t.Context())+ require.NoError(t, err)++ _, err = cs.AccessToken(t.Context())+ require.NoError(t, err)++ require.Equal(t, int64(1), hits.Load())+ })++ t.Run("expire", func(t *testing.T) {+ var hits atomic.Int64+ server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "token": "test-token",+ "expires_at": "2099-01-01T00:00:00Z",+ })+ })+ defer server.Close()++ cs, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 100)+ require.NoError(t, err)++ _, err = cs.AccessToken(t.Context())+ require.NoError(t, err)++ token := cs.accessTokenIfValid()+ require.NotNil(t, token)++ token.ExpiresAt = &github.Timestamp{Time: time.Now()}++ token = cs.accessTokenIfValid()+ require.Nil(t, token)++ // race these for good measure+ var wg sync.WaitGroup+ for range 3 {+ wg.Go(func() {+ _, err := cs.AccessToken(t.Context())+ require.NoError(t, err)+ })+ }+ wg.Wait()++ token = cs.accessTokenIfValid()+ require.NotNil(t, token)++ require.Equal(t, int64(2), hits.Load())+ })+}diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.goindex 0739d4c9a..027d967ae 100644--- a/pkg/sources/github/connector_app.go+++ b/pkg/sources/github/connector_app.go@@ -2,19 +2,13 @@ package githubimport ("fmt"- "net/http"- "net/url""strconv"- "strings""sync"- "time"- "github.com/bradleyfalzon/ghinstallation/v2"gogit "github.com/go-git/go-git/v5""github.com/google/go-github/v67/github""github.com/shurcooL/githubv4"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"@@ -24,13 +18,10 @@ type appConnector struct {installationClient *github.ClientinstallationID int64appID int64- appPrivateKey []byteapiEndpoint string+ appPrivateKey stringmu sync.RWMutex- // clientsByInstallationID is scoped to this connector/source run and is- // bounded by the installations touched while scanning.- clientsByInstallationID map[int64]*appInstallationClients// repoInstallationMap maps repo clone URLs to their owning installation// ID for repos discovered from non-default installations. Clone checks@@ -38,20 +29,6 @@ type appConnector struct {repoInstallationMap map[string]int64}-type appInstallationClients struct {- apiClient *github.Client- graphqlClient *githubv4.Client- tokenMu sync.RWMutex- token *github.InstallationToken-}--func (cs *appInstallationClients) accessTokenIfValid() *github.InstallationToken {- if cs.token == nil || cs.token.GetExpiresAt().Before(time.Now()) {- return nil- }- return cs.token-}-var _ Connector = (*appConnector)(nil)const githubHTTPTimeoutSeconds = 60@@ -76,23 +53,18 @@ func NewAppConnector(apiEndpoint string, app *credentialspb.GitHubApp, scanAllInreturn nil, fmt.Errorf("could not parse app ID %q: %w", app.AppId, err)}- installationTransport, err := newAppsTransport(apiEndpoint, appID, []byte(app.PrivateKey))- if err != nil {- return nil, fmt.Errorf("could not create installation client transport: %w", err)- }- installationClient, err := newGitHubClientWithTransport(apiEndpoint, installationTransport)+ installationClient, err := GetAppClient(context.Background(), apiEndpoint, appID, app.PrivateKey)if err != nil {return nil, fmt.Errorf("could not create installation client: %w", err)}connector := &appConnector{- installationClient: installationClient,- installationID: installationID,- appID: appID,- appPrivateKey: []byte(app.PrivateKey),- apiEndpoint: apiEndpoint,- clientsByInstallationID: make(map[int64]*appInstallationClients),- repoInstallationMap: make(map[string]int64),+ installationClient: installationClient,+ installationID: installationID,+ appID: appID,+ apiEndpoint: apiEndpoint,+ appPrivateKey: app.PrivateKey,+ repoInstallationMap: make(map[string]int64),}// When no default installation is configured (installationID == 0, only@@ -149,7 +121,7 @@ func (c *appConnector) graphqlClientForInstallation(installID int64) (*githubv4.return nil, err}- return clients.graphqlClient, nil+ return clients.GraphQLClient, nil}func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...string) (string, *gogit.Repository, error) {@@ -164,57 +136,17 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string, args ...stringreturn "", nil, fmt.Errorf("no GitHub App installation resolved for repo %q; set githubApp.installationId to a fallback installation to scan repos outside installation listings (e.g. member repos with scanUsers) together with scanAllInstallations", repoURL)}- token, err := c.accessTokenForInstallation(ctx, installID)- if err != nil {- return "", nil, err- }-- return git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...)-}--// accessTokenForInstallation returns an installation access token.-func (c *appConnector) accessTokenForInstallation(ctx context.Context, installID int64) (string, error) {- if installID == 0 {- return "", fmt.Errorf("tried to fetch installation access token for id 0")- }-- clients, err := c.clientsForInstallation(installID)+ cs, err := c.clientsForInstallation(installID)if err != nil {- return "", err- }-- clients.tokenMu.RLock()- token := clients.accessTokenIfValid()- clients.tokenMu.RUnlock()-- if token != nil {- return token.GetToken(), nil- }-- clients.tokenMu.Lock()- defer clients.tokenMu.Unlock()-- if token := clients.accessTokenIfValid(); token != nil {- return token.GetToken(), nil+ return "", nil, fmt.Errorf("cannot fetch installation clients for installation %d: %w", installID, err)}- token, _, err = c.installationClient.Apps.CreateInstallationToken(- ctx,- installID,- &github.InstallationTokenOptions{},- )+ token, err := cs.AccessToken(ctx)if err != nil {- return "", fmt.Errorf("could not create installation token for installation %d: %w", installID, err)- }-- if exp := token.GetExpiresAt(); !exp.IsZero() {- // Refresh just before the real expiry- token.ExpiresAt = &github.Timestamp{Time: exp.Add(-time.Minute)}+ return "", nil, fmt.Errorf("could not fetch installation access token for installation %d: %w", installID, err)}- clients.token = token-- return token.GetToken(), nil+ return git.CloneRepoUsingToken(ctx, token, repoURL, "", "x-access-token", true, args...)}// installationIDForRepo returns the mapped installation ID for repoURL. When no@@ -309,135 +241,9 @@ func (c *appConnector) APIClientForInstallation(installationID int64) (*github.Creturn nil, err}- return clients.apiClient, nil-}--func (c *appConnector) clientsForInstallation(installationID int64) (*appInstallationClients, error) {- c.mu.RLock()- clients, ok := c.clientsByInstallationID[installationID]- c.mu.RUnlock()- if ok {- return clients, nil- }-- clients, err := c.clientsForInstallationLocked(installationID)- if err != nil {- return nil, err- }-- return clients, nil+ return clients.APIClient, nil}-func (c *appConnector) clientsForInstallationLocked(installationID int64) (*appInstallationClients, error) {- c.mu.Lock()- defer c.mu.Unlock()-- if clients, ok := c.clientsByInstallationID[installationID]; ok {- return clients, nil- }-- clients, err := c.createAPIClientsForInstallation(installationID)- if err != nil {- return nil, err- }-- c.clientsByInstallationID[installationID] = clients-- return clients, nil-}--// createAPIClientsForInstallation creates both types of clients at once and-// saves along with the transport in an appInstallationClients, so those-// structs are never in a partially-filled state.-func (c *appConnector) createAPIClientsForInstallation(installationID int64) (*appInstallationClients, error) {- appsTransport, err := newAppsTransport(c.apiEndpoint, c.appID, c.appPrivateKey)- if err != nil {- return nil, fmt.Errorf("could not create app transport for installation %d: %w", installationID, err)- }-- // NewFromAppsTransport inherits BaseURL from appsTransport, which- // newAppsTransport already set correctly (incl. GHE.com). Don't override it- // with the raw endpoint here or GHE.com token refresh would 401.- transport := ghinstallation.NewFromAppsTransport(appsTransport, installationID)-- apiClient, err := newGitHubClientWithTransport(c.apiEndpoint, transport)- if err != nil {- return nil, fmt.Errorf("could not create API client for installation %d: %w", installationID, err)- }-- // the only thing this uses context for is a level 2 log line...- gqlClient, err := createGraphqlClient(context.Background(), apiClient.Client(), c.apiEndpoint)- if err != nil {- return nil, err- }-- return &appInstallationClients{- apiClient: apiClient,- graphqlClient: gqlClient,- }, nil-}--func newAppsTransport(apiEndpoint string, appID int64, privateKey []byte) (*ghinstallation.AppsTransport, error) {- httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)- appsTransport, err := ghinstallation.NewAppsTransport(httpClient.Transport, appID, privateKey)- if err != nil {- return nil, err- }-- // ghinstallation uses BaseURL to build the token exchange URL- // {BaseURL}/app/installations/{id}/access_tokens. For GHE.com this must be- // the api.SUBDOMAIN.ghe.com base with no /api/v3 and no trailing slash- // (which ghinstallation does not expect), otherwise token refresh 401s.- baseURL, err := appsBaseURL(apiEndpoint)- if err != nil {- return nil, err- }- appsTransport.BaseURL = strings.TrimRight(baseURL, "/")- return appsTransport, nil-}--// appsBaseURL returns the BaseURL that ghinstallation transports should use-// for token exchange/refresh. For GHE.com it resolves to the api.* subdomain;-// for what is likely GHES, it ensures /api/v3 is used; and for anything else-// including github.com it is the endpoint as-is.-func appsBaseURL(apiEndpoint string) (string, error) {- if isGHECloud(apiEndpoint) {- normalizedURL, err := normalizeGHECloudAPIEndpoint(apiEndpoint)- if err != nil {- return "", fmt.Errorf("could not normalize GHE.com endpoint: %w", err)- }- return normalizedURL, nil- }-- u, err := url.Parse(apiEndpoint)- if err != nil {- return "", fmt.Errorf("malformed endpoint url: %w", err)- }- h := u.Hostname()-- if !strings.HasSuffix(u.Path, "/") {- u.Path += "/"- }-- // If we're not dealing with github.com or GHE, then maybe GHES, which- // needs /api/v3 for ghinstallation find the right place- if !strings.HasPrefix(h, "api.") && !strings.Contains(h, ".api.") {- if !strings.HasSuffix(u.Path, "/api/v3/") {- u.Path += "api/v3/"- }- }-- return u.String(), nil-}--func newGitHubClientWithTransport(apiEndpoint string, transport http.RoundTripper) (*github.Client, error) {- httpClient := common.RetryableHTTPClientTimeout(githubHTTPTimeoutSeconds)- httpClient.Transport = transport-- // GHE.com (GHEC with data residency) serves its REST API at the root of- // api.SUBDOMAIN.ghe.com (like api.github.com), NOT under the GHES /api/v3 path.- if isGHECloud(apiEndpoint) {- return createGHECloudClient(httpClient, apiEndpoint)- }- return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint)+func (c *appConnector) clientsForInstallation(installationID int64) (*AppInstallationClients, error) {+ return GetAppInstallationClients(context.Background(), c.apiEndpoint, c.appID, c.appPrivateKey, installationID)}diff --git a/pkg/sources/github/connector_app_test.go b/pkg/sources/github/connector_app_test.goindex 2d5898395..a0ac2fe6c 100644--- a/pkg/sources/github/connector_app_test.go+++ b/pkg/sources/github/connector_app_test.go@@ -10,9 +10,7 @@ import ("net/http/httptest""strings""sync"- "sync/atomic""testing"- "time""github.com/google/go-github/v67/github""github.com/shurcooL/githubv4"@@ -58,19 +56,24 @@ func TestAPIClientForInstallation(t *testing.T) {}))defer server.Close()- connector := &appConnector{- appID: 12345,- appPrivateKey: privKey,- apiEndpoint: server.URL,- clientsByInstallationID: make(map[int64]*appInstallationClients),- }+ connector, err := NewAppConnector(+ server.URL,+ &credentialspb.GitHubApp{+ PrivateKey: string(privKey),+ AppId: "12345",+ InstallationId: "42",+ },+ false,+ )+ require.NoError(t, err)+ ac := connector.(*appConnector)t.Run("creates distinct clients for different installations", func(t *testing.T) {- client1, err := connector.APIClientForInstallation(111)+ client1, err := ac.APIClientForInstallation(111)require.NoError(t, err)assert.NotNil(t, client1)- client2, err := connector.APIClientForInstallation(222)+ client2, err := ac.APIClientForInstallation(222)require.NoError(t, err)assert.NotNil(t, client2)@@ -78,9 +81,9 @@ func TestAPIClientForInstallation(t *testing.T) {})t.Run("reuses clients for the same installation", func(t *testing.T) {- client1, err := connector.APIClientForInstallation(333)+ client1, err := ac.APIClientForInstallation(333)require.NoError(t, err)- client2, err := connector.APIClientForInstallation(333)+ client2, err := ac.APIClientForInstallation(333)require.NoError(t, err)assert.Same(t, client1, client2)@@ -91,7 +94,7 @@ func TestAPIClientForInstallation(t *testing.T) {tokenRequestPaths = nilmu.Unlock()- client, err := connector.APIClientForInstallation(42)+ client, err := ac.APIClientForInstallation(42)require.NoError(t, err)ctx := trContext.Background()@@ -110,11 +113,11 @@ func TestAPIClientForInstallation(t *testing.T) {ctx := trContext.Background()- client1, err := connector.APIClientForInstallation(100)+ client1, err := ac.APIClientForInstallation(100)require.NoError(t, err)_, _, _ = client1.Organizations.ListMembers(ctx, "org-a", nil)- client2, err := connector.APIClientForInstallation(200)+ client2, err := ac.APIClientForInstallation(200)require.NoError(t, err)_, _, _ = client2.Organizations.ListMembers(ctx, "org-b", nil)@@ -156,6 +159,7 @@ func TestNewAppConnectorDefaultAPIClientUsesConfiguredInstallation(t *testing.T)AppId: "12345",}, false)require.NoError(t, err)+require.NotNil(t, connector.APIClient())require.NotNil(t, connector.GraphQLClient())@@ -235,9 +239,8 @@ func TestAPIClientAndGraphQLClientNonNilWithoutDefaultInstallation(t *testing.T)// error instead of attempting to use installation ID 0.func TestCloneErrorsWithoutResolvedInstallation(t *testing.T) {connector := &appConnector{- installationID: 0,- clientsByInstallationID: make(map[int64]*appInstallationClients),- repoInstallationMap: make(map[string]int64),+ installationID: 0,+ repoInstallationMap: make(map[string]int64),}_, _, err := connector.Clone(trContext.Background(), "https://github.com/some-member/personal-repo.git")@@ -245,185 +248,6 @@ func TestCloneErrorsWithoutResolvedInstallation(t *testing.T) {assert.Contains(t, err.Error(), "no GitHub App installation resolved")}-func TestAccessTokenForInstallation(t *testing.T) {- privKey := generateTestPrivateKey(t)-- accessTokenServer := func(accessTokenHandler func(w http.ResponseWriter, r *http.Request)) *httptest.Server {- return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- w.Header().Set("Content-Type", "application/json")-- if r.Method == "POST" && strings.Contains(r.URL.Path, "access_tokens") {- accessTokenHandler(w, r)- return- }-- // Default: return empty JSON array for any list endpoint.- _ = json.NewEncoder(w).Encode([]interface{}{})- }))- }-- t.Run("invalid", func(t *testing.T) {- _, err := (&appConnector{}).accessTokenForInstallation(trContext.Background(), 0)- require.Error(t, err)- })-- t.Run("success", func(t *testing.T) {- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {- _ = json.NewEncoder(w).Encode(map[string]interface{}{- "token": "test-token",- "expires_at": "2099-01-01T00:00:00Z",- })- })- defer server.Close()-- c, err := NewAppConnector(- server.URL,- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},- false,- )- require.NoError(t, err)- ac := c.(*appConnector)-- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)- require.NoError(t, err)- })-- t.Run("rate limit", func(t *testing.T) {- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {- w.Header().Add("X-RateLimit-Remaining", "0")- w.WriteHeader(http.StatusForbidden)-- _ = json.NewEncoder(w).Encode(map[string]interface{}{})- })- defer server.Close()-- c, err := NewAppConnector(- server.URL,- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},- false,- )- require.NoError(t, err)- ac := c.(*appConnector)-- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)- require.Error(t, err)-- var gherr *github.RateLimitError- require.ErrorAs(t, err, &gherr)- })-- t.Run("not found / no installation", func(t *testing.T) {- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {- w.WriteHeader(http.StatusNotFound)-- _ = json.NewEncoder(w).Encode(map[string]interface{}{})- })- defer server.Close()-- c, err := NewAppConnector(- server.URL,- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},- false,- )- require.NoError(t, err)- ac := c.(*appConnector)-- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)- require.Error(t, err)- })-- t.Run("reuse", func(t *testing.T) {- var hits atomic.Int64-- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {- hits.Add(1)- _ = json.NewEncoder(w).Encode(map[string]interface{}{- "token": "test-token",- "expires_at": "2099-01-01T00:00:00Z",- })- })- defer server.Close()-- c, err := NewAppConnector(- server.URL,- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},- false,- )- require.NoError(t, err)- ac := c.(*appConnector)-- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)- require.NoError(t, err)-- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)- require.NoError(t, err)-- require.Equal(t, int64(1), hits.Load())- })-- t.Run("expire", func(t *testing.T) {- var hits atomic.Int64- server := accessTokenServer(func(w http.ResponseWriter, r *http.Request) {- hits.Add(1)- _ = json.NewEncoder(w).Encode(map[string]interface{}{- "token": "test-token",- "expires_at": "2099-01-01T00:00:00Z",- })- })- defer server.Close()-- c, err := NewAppConnector(- server.URL,- &credentialspb.GitHubApp{PrivateKey: string(privKey), AppId: "12345", InstallationId: "100"},- false,- )- require.NoError(t, err)- ac := c.(*appConnector)-- _, err = ac.accessTokenForInstallation(trContext.Background(), 100)- require.NoError(t, err)-- cs, err := ac.clientsForInstallation(100)- require.NoError(t, err)- token := cs.accessTokenIfValid()- require.NotNil(t, token)-- token.ExpiresAt = &github.Timestamp{Time: time.Now()}-- token = cs.accessTokenIfValid()- require.Nil(t, token)-- // race these for good measure- var wg sync.WaitGroup- for range 3 {- wg.Go(func() {- _, err := ac.accessTokenForInstallation(trContext.Background(), 100)- require.NoError(t, err)- })- }- wg.Wait()-- token = cs.accessTokenIfValid()- require.NotNil(t, token)-- require.Equal(t, int64(2), hits.Load())- })-}--func TestAPIClientForInstallationUsesConfiguredClient(t *testing.T) {- defaultClient := github.NewClient(nil)- connector := &appConnector{- installationID: 100,- clientsByInstallationID: map[int64]*appInstallationClients{- 100: {apiClient: defaultClient},- },- }-- got, err := connector.APIClientForInstallation(100)- require.NoError(t, err)- assert.Same(t, defaultClient, got)-}-func TestCloneUsesRepoInstallationMap(t *testing.T) {connector := &appConnector{installationID: 100,@@ -475,19 +299,23 @@ func TestAPIClientForRepoUsesRepoInstallationMap(t *testing.T) {}))defer server.Close()- connector := &appConnector{- installationID: 100,- appID: 12345,- appPrivateKey: privKey,- apiEndpoint: server.URL,- clientsByInstallationID: make(map[int64]*appInstallationClients),- repoInstallationMap: make(map[string]int64),- }- connector.setRepoInstallation("https://github.com/other-org/repo.git", 999)+ connector, err := NewAppConnector(+ server.URL,+ &credentialspb.GitHubApp{+ PrivateKey: string(privKey),+ AppId: "12345",+ InstallationId: "100",+ },+ false,+ )+ require.NoError(t, err)+ ac := connector.(*appConnector)++ ac.setRepoInstallation("https://github.com/other-org/repo.git", 999)- client, err := connector.APIClientForRepo("https://github.com/other-org/repo.git")+ client, err := ac.APIClientForRepo("https://github.com/other-org/repo.git")require.NoError(t, err)- cachedClient, err := connector.APIClientForRepo("https://github.com/other-org/repo.git")+ cachedClient, err := ac.APIClientForRepo("https://github.com/other-org/repo.git")require.NoError(t, err)assert.Same(t, client, cachedClient)@@ -509,44 +337,56 @@ func TestGraphQLClientForRepoCachesClients(t *testing.T) {defer server.Close()defaultGraphQLClient := &githubv4.Client{}- connector := &appConnector{- installationID: 100,- appID: 12345,- appPrivateKey: privKey,- apiEndpoint: server.URL,- clientsByInstallationID: map[int64]*appInstallationClients{- 100: {graphqlClient: defaultGraphQLClient},++ connector, err := NewAppConnector(+ server.URL,+ &credentialspb.GitHubApp{+ PrivateKey: string(privKey),+ AppId: "12345",+ InstallationId: "100",},- repoInstallationMap: make(map[string]int64),- }- connector.setRepoInstallation("https://github.com/other-org/repo.git", 999)+ false,+ )+ require.NoError(t, err)+ ac := connector.(*appConnector)++ clientsByInstallationID.Set(+ appInstallationClientKey{+ appClientKey{server.URL, 12345, string(privKey)},+ 100,+ },+ &AppInstallationClients{+ InstallationID: 100,+ GraphQLClient: defaultGraphQLClient,+ },+ )+ ac.setRepoInstallation("https://github.com/other-org/repo.git", 999)ctx := trContext.Background()t.Run("returns default client for default installation repos", func(t *testing.T) {- got, err := connector.GraphQLClientForRepo(ctx, "https://github.com/default-org/repo.git")+ got, err := ac.GraphQLClientForRepo(ctx, "https://github.com/default-org/repo.git")require.NoError(t, err)assert.Same(t, defaultGraphQLClient, got)})t.Run("caches clients for non-default installations", func(t *testing.T) {- client1, err := connector.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")+ client1, err := ac.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")require.NoError(t, err)assert.NotSame(t, defaultGraphQLClient, client1)- client2, err := connector.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")+ client2, err := ac.GraphQLClientForRepo(ctx, "https://github.com/other-org/repo.git")require.NoError(t, err)assert.Same(t, client1, client2)- apiClient, err := connector.APIClientForRepo("https://github.com/other-org/repo.git")+ apiClient, err := ac.APIClientForRepo("https://github.com/other-org/repo.git")require.NoError(t, err)- connector.mu.RLock()- cachedClients := connector.clientsByInstallationID[999]- connector.mu.RUnlock()+ cachedClients, err := GetAppInstallationClients(t.Context(), server.URL, 12345, string(privKey), 999)+ require.NoError(t, err)require.NotNil(t, cachedClients)- assert.Same(t, apiClient, cachedClients.apiClient)- assert.Same(t, client1, cachedClients.graphqlClient)+ assert.Same(t, apiClient, cachedClients.APIClient)+ assert.Same(t, client1, cachedClients.GraphQLClient)})}diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.goindex 29911c078..dad98fee8 100644--- a/pkg/sources/github/github_test.go+++ b/pkg/sources/github/github_test.go@@ -327,7 +327,8 @@ func TestAddMembersByApp(t *testing.T) {InstallationId: "1337",AppId: "4141",},- }})+ },+ })err := s.addMembersByApp(context.Background(), s.connector.(*appConnector), noopReporter())assert.Nil(t, err)assert.Equal(t, 3, len(s.memberCache))@@ -434,7 +435,8 @@ func TestValidate_ScanAllInstallationsWithoutDefaultInstallation(t *testing.T) {PrivateKey: createPrivateKey(),AppId: "4141",},- }})+ },+ })errs := s.Validate(context.Background())assert.Empty(t, errs)@@ -2330,11 +2332,8 @@ func Test_ScanMultipleTargets_MultipleErrors(t *testing.T) {func TestScanAllInstallationsTargetMappingErrorIsTargetedScanError(t *testing.T) {s := &Source{- conn: &sourcespb.GitHub{ScanAllInstallations: true},- connector: &appConnector{- clientsByInstallationID: make(map[int64]*appInstallationClients),- repoInstallationMap: make(map[string]int64),- },+ conn: &sourcespb.GitHub{ScanAllInstallations: true},+ connector: &appConnector{},}ctx := context.Background()chunksChan := make(chan *sources.Chunk)