Skip to content

Find, verify, and analyze leaked credentials

Carry GitHub App installation ID on repo units (#5215)

Bill Rich20652fbbdeff
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index b691d24f4..61170789a 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -1,6 +1,7 @@
package github
import (
+ "encoding/json"
"errors"
"fmt"
"io"
@@ -81,7 +82,6 @@ type Source struct {
commentsTimeframeDays uint32
sources.Progress
- sources.CommonSourceUnitUnmarshaller
useAuthInUrl bool // pass credentials in the repository urls for cloning
}
@@ -97,6 +97,13 @@ var _ sources.SourceUnit = (*GistUnit)(nil)
type RepoUnit struct {
Name string `json:"name"`
URL string `json:"url"`
+ // InstallationID is the GitHub App installation that owns this repo, set
+ // during scan-all-installations enumeration. Carrying it on the unit lets
+ // ChunkUnit clone with the correct installation token without re-deriving
+ // the mapping, which otherwise re-lists every installation's repos on every
+ // scanned unit (see INT-790). Zero when the source is not an installation
+ // scan or the unit predates this field.
+ InstallationID int64 `json:"installation_id,omitempty"`
}
func (r RepoUnit) SourceUnitID() (string, sources.SourceUnitKind) { return r.URL, "repo" }
@@ -110,6 +117,66 @@ type GistUnit struct {
func (g GistUnit) SourceUnitID() (string, sources.SourceUnitKind) { return g.URL, "gist" }
func (g GistUnit) Display() string { return g.Name }
+// unitEnvelope is just the JSON equivalent of apipb.SourceUnit, so we can
+// unmarshal source unit payloads.
+type unitEnvelope struct {
+ ID string `json:"id"`
+ Kind sources.SourceUnitKind `json:"kind,omitempty"`
+ Name string `json:"display,omitempty"`
+ UnitData []byte `json:"unit_data,omitempty"`
+}
+
+func (u unitEnvelope) SourceUnitID() (string, sources.SourceUnitKind) { return u.ID, u.Kind }
+func (u unitEnvelope) Display() string { return u.Name }
+
+func unmarshalSourceUnit[unitType sources.SourceUnit](data []byte) (unitType, error) {
+ u := new(unitType)
+
+ if err := json.Unmarshal(data, u); err != nil {
+ return *new(unitType), err
+ }
+
+ return *u, nil
+}
+
+func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
+ env, err := unmarshalSourceUnit[unitEnvelope](data)
+ if err != nil {
+ return nil, err
+ }
+
+ switch env.Kind {
+ case "": // data is a raw SourceUnit, no envelope
+ // Look for a raw RepoUnit first
+ ru, err := unmarshalSourceUnit[RepoUnit](data)
+ if err != nil {
+ return nil, err
+ }
+
+ if ru.URL != "" { // This is the test for a valid RepoUnit
+ return ru, nil
+ }
+
+ if env.ID == "" { // This is the test for a valid generic GitHub Source Unit
+ return nil, errors.New("not a github source unit")
+ }
+
+ return env, nil
+ case "repo":
+ if len(env.UnitData) > 0 {
+ return unmarshalSourceUnit[RepoUnit](env.UnitData)
+ }
+ return RepoUnit{Name: env.Name, URL: env.ID}, nil
+ case "gist":
+ if len(env.UnitData) > 0 {
+ return unmarshalSourceUnit[GistUnit](env.UnitData)
+ }
+ return GistUnit{Name: env.Name, URL: env.ID}, nil
+ default:
+ return nil, fmt.Errorf("unrecognized source unit kind %q", env.Kind)
+ }
+}
+
// --------------------------------------------------------------------------------
// WithCustomContentWriter sets the useCustomContentWriter flag on the source.
@@ -433,7 +500,16 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e
}
continue
}
- if err := dedupeReporter.UnitOk(ctx, RepoUnit{Name: name, URL: url}); err != nil {
+
+ var installationID int64 = 0
+ if ac, ok := s.connector.(*appConnector); ok {
+ ac.ensureRepoInstallation(url, name)
+ installationID, _ = ac.installationIDForRepo(url)
+ }
+
+ ru := RepoUnit{Name: name, URL: url, InstallationID: installationID}
+
+ if err := dedupeReporter.UnitOk(ctx, ru); err != nil {
return err
}
}
@@ -2428,8 +2504,14 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte
repoURL, kind := unit.SourceUnitID()
ctx = context.WithValue(ctx, "repo", repoURL)
- if connector, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations && kind == "repo" {
- if err := s.mapReposToInstallations(ctx, connector, []string{repoURL}); err != nil {
+ // If the unit has an installation ID, use it. Fetching it is slow,
+ // specifically it stalls large multi-org scans on rate limits (INT-790).
+
+ // [CG] This is pretty ugly; if you can clean it up please do (I failed).
+ if ac, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations && kind == "repo" {
+ if ru, ok := unit.(RepoUnit); ok && ru.InstallationID > 0 {
+ ac.setRepoInstallationForRepoName(repoURL, ru.Name, ru.InstallationID)
+ } else if err := s.mapReposToInstallations(ctx, ac, []string{repoURL}); err != nil {
return err
}
}
diff --git a/pkg/sources/github/map_installations_test.go b/pkg/sources/github/map_installations_test.go
new file mode 100644
index 000000000..51c4fda51
--- /dev/null
+++ b/pkg/sources/github/map_installations_test.go
@@ -0,0 +1,167 @@
+package github
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/google/go-github/v67/github"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+// This is meant to track APIUnitReporter.UnitOK in scanner/cmd/scanner/pipeline.go
+// precisely
+func newMarshalledRepoUnit(t *testing.T, su sources.SourceUnit) []byte {
+ t.Helper()
+
+ id, kind := su.SourceUnitID()
+
+ unitData, err := json.Marshal(su)
+ require.NoError(t, err)
+
+ out, err := json.Marshal(map[string]any{
+ "id": id,
+ "kind": string(kind),
+ "display": su.Display(),
+ "unit_data": unitData,
+ })
+ require.NoError(t, err)
+
+ return out
+}
+
+func TestGitHub_UnmarshalSourceUnit(t *testing.T) {
+ s := &Source{}
+
+ t.Run("source unit with unit_data recovers installation id", func(t *testing.T) {
+ data := newMarshalledRepoUnit(t, RepoUnit{
+ Name: "spotify/backstage",
+ URL: "https://github.com/spotify/backstage.git",
+ InstallationID: 2448,
+ })
+
+ unit, err := s.UnmarshalSourceUnit(data)
+ require.NoError(t, err)
+
+ repoUnit, ok := unit.(RepoUnit)
+ require.True(t, ok, "expected a RepoUnit so ChunkUnit can read the installation id")
+ assert.Equal(t, "https://github.com/spotify/backstage.git", repoUnit.URL)
+ assert.Equal(t, "spotify/backstage", repoUnit.Name)
+ assert.Equal(t, int64(2448), repoUnit.InstallationID)
+
+ id, kind := unit.SourceUnitID()
+ assert.Equal(t, "https://github.com/spotify/backstage.git", id)
+ assert.EqualValues(t, "repo", kind)
+ })
+
+ t.Run("legacy source unit without unit_data has no installation id", func(t *testing.T) {
+ data, err := json.Marshal(map[string]any{
+ "id": "https://github.com/acme/widgets.git",
+ "kind": "repo",
+ "display": "acme/widgets",
+ })
+ require.NoError(t, err)
+
+ unit, err := s.UnmarshalSourceUnit(data)
+ require.NoError(t, err)
+
+ repoUnit, ok := unit.(RepoUnit)
+ require.True(t, ok)
+ assert.Equal(t, "https://github.com/acme/widgets.git", repoUnit.URL)
+ assert.Equal(t, "acme/widgets", repoUnit.Name)
+ assert.Zero(t, repoUnit.InstallationID, "legacy units fall back to deriving the installation")
+ })
+
+ t.Run("full enumeration payload", func(t *testing.T) {
+ data, err := json.Marshal(RepoUnit{
+ Name: "acme/widgets",
+ URL: "https://github.com/acme/widgets.git",
+ InstallationID: 111,
+ })
+ require.NoError(t, err)
+
+ unit, err := s.UnmarshalSourceUnit(data)
+ require.NoError(t, err)
+
+ repoUnit, ok := unit.(RepoUnit)
+ require.True(t, ok)
+ assert.Equal(t, int64(111), repoUnit.InstallationID)
+ })
+
+ t.Run("gist unit round trips", func(t *testing.T) {
+ data := newMarshalledRepoUnit(t, GistUnit{Name: "abc", URL: "https://gist.github.com/abc.git"})
+
+ unit, err := s.UnmarshalSourceUnit(data)
+ require.NoError(t, err)
+
+ _, ok := unit.(GistUnit)
+ require.True(t, ok)
+ _, kind := unit.SourceUnitID()
+ assert.EqualValues(t, "gist", kind)
+ })
+
+ t.Run("rejects unrecognized payload", func(t *testing.T) {
+ _, err := s.UnmarshalSourceUnit([]byte(`{"unrelated":"value"}`))
+ require.Error(t, err)
+ })
+}
+
+// Ensure we store installation IDs when enumerating
+func TestGitHub_ProcessReposEmitsInstallationID(t *testing.T) {
+ ctx := context.Background()
+
+ connector := &appConnector{
+ installationID: 1,
+ repoInstallationMap: make(map[string]int64),
+ }
+ s := &Source{
+ conn: &sourcespb.GitHub{ScanAllInstallations: true},
+ connector: connector,
+ filteredRepoCache: &filteredRepoCache{Cache: simple.NewCache[string]()},
+ repoInfoCache: newRepoInfoCache(),
+ }
+
+ const installID int64 = 2448
+ repo := &github.Repository{
+ Name: github.String("backstage"),
+ FullName: github.String("spotify/backstage"),
+ CloneURL: github.String("https://github.com/spotify/backstage.git"),
+ Owner: &github.User{Login: github.String("spotify"), Type: github.String("Organization")},
+ }
+
+ // Mirror enumerateAllInstallationRepos: the lister records the installation
+ // for each repo in a page before processRepos emits the units for it.
+ page := 0
+ listRepos := func(ctx context.Context, _ string, _ repoListOptions) ([]*github.Repository, *github.Response, error) {
+ page++
+ if page > 1 {
+ return nil, &github.Response{}, nil
+ }
+ connector.setRepoInstallationForRepoName(repo.GetCloneURL(), repo.GetName(), installID)
+ return []*github.Repository{repo}, &github.Response{}, nil
+ }
+
+ var units []sources.SourceUnit
+ reporter := sources.VisitorReporter{
+ VisitUnit: func(_ context.Context, u sources.SourceUnit) error {
+ units = append(units, u)
+ return nil
+ },
+ }
+
+ err := s.processRepos(ctx, "spotify", reporter, listRepos, &appListOptions{
+ ListOptions: github.ListOptions{PerPage: 100},
+ })
+ require.NoError(t, err)
+
+ require.Len(t, units, 1)
+ repoUnit, ok := units[0].(RepoUnit)
+ require.True(t, ok)
+ assert.Equal(t, "https://github.com/spotify/backstage.git", repoUnit.URL)
+ assert.Equal(t, installID, repoUnit.InstallationID)
+}
diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go
index 6a169233a..5ac878a8c 100644
--- a/pkg/sources/github/repo.go
+++ b/pkg/sources/github/repo.go
@@ -300,14 +300,22 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourc
s.totalRepoSize += r.GetSize()
s.filteredRepoCache.Set(repoName, repoURL)
s.cacheRepoInfo(r)
+
+ unit := RepoUnit{Name: repoName, URL: repoURL}
// Repos enumerated outside installation listings (e.g. member
// personal repos) belong to no installation; map them to the
// default installation so the scan-all mapping in ChunkUnit
// doesn't reject repos we just enumerated.
if connector, ok := s.connector.(*appConnector); ok && s.conn.GetScanAllInstallations() {
connector.ensureRepoInstallation(repoURL, r.GetName())
+ // Record the resolved installation on the unit so ChunkUnit
+ // scans it with the correct token without re-listing every
+ // installation's repos (INT-790).
+ if installID, mapped := connector.installationIDForRepo(repoURL); mapped {
+ unit.InstallationID = installID
+ }
}
- if err := reporter.UnitOk(ctx, RepoUnit{Name: repoName, URL: repoURL}); err != nil {
+ if err := reporter.UnitOk(ctx, unit); err != nil {
return err
}