Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Carry GitHub App installation ID on repo units (#5215)
Bill Rich20652fbbdeff
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex b691d24f4..61170789a 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -1,6 +1,7 @@package githubimport (+ "encoding/json""errors""fmt""io"@@ -81,7 +82,6 @@ type Source struct {commentsTimeframeDays uint32sources.Progress- sources.CommonSourceUnitUnmarshalleruseAuthInUrl bool // pass credentials in the repository urls for cloning}@@ -97,6 +97,13 @@ var _ sources.SourceUnit = (*GistUnit)(nil)type RepoUnit struct {Name string `json:"name"`URL string `json:"url"`+ // InstallationID is the GitHub App installation that owns this repo, set+ // during scan-all-installations enumeration. Carrying it on the unit lets+ // ChunkUnit clone with the correct installation token without re-deriving+ // the mapping, which otherwise re-lists every installation's repos on every+ // scanned unit (see INT-790). Zero when the source is not an installation+ // scan or the unit predates this field.+ InstallationID int64 `json:"installation_id,omitempty"`}func (r RepoUnit) SourceUnitID() (string, sources.SourceUnitKind) { return r.URL, "repo" }@@ -110,6 +117,66 @@ type GistUnit struct {func (g GistUnit) SourceUnitID() (string, sources.SourceUnitKind) { return g.URL, "gist" }func (g GistUnit) Display() string { return g.Name }+// unitEnvelope is just the JSON equivalent of apipb.SourceUnit, so we can+// unmarshal source unit payloads.+type unitEnvelope struct {+ ID string `json:"id"`+ Kind sources.SourceUnitKind `json:"kind,omitempty"`+ Name string `json:"display,omitempty"`+ UnitData []byte `json:"unit_data,omitempty"`+}++func (u unitEnvelope) SourceUnitID() (string, sources.SourceUnitKind) { return u.ID, u.Kind }+func (u unitEnvelope) Display() string { return u.Name }++func unmarshalSourceUnit[unitType sources.SourceUnit](data []byte) (unitType, error) {+ u := new(unitType)++ if err := json.Unmarshal(data, u); err != nil {+ return *new(unitType), err+ }++ return *u, nil+}++func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {+ env, err := unmarshalSourceUnit[unitEnvelope](data)+ if err != nil {+ return nil, err+ }++ switch env.Kind {+ case "": // data is a raw SourceUnit, no envelope+ // Look for a raw RepoUnit first+ ru, err := unmarshalSourceUnit[RepoUnit](data)+ if err != nil {+ return nil, err+ }++ if ru.URL != "" { // This is the test for a valid RepoUnit+ return ru, nil+ }++ if env.ID == "" { // This is the test for a valid generic GitHub Source Unit+ return nil, errors.New("not a github source unit")+ }++ return env, nil+ case "repo":+ if len(env.UnitData) > 0 {+ return unmarshalSourceUnit[RepoUnit](env.UnitData)+ }+ return RepoUnit{Name: env.Name, URL: env.ID}, nil+ case "gist":+ if len(env.UnitData) > 0 {+ return unmarshalSourceUnit[GistUnit](env.UnitData)+ }+ return GistUnit{Name: env.Name, URL: env.ID}, nil+ default:+ return nil, fmt.Errorf("unrecognized source unit kind %q", env.Kind)+ }+}+// --------------------------------------------------------------------------------// WithCustomContentWriter sets the useCustomContentWriter flag on the source.@@ -433,7 +500,16 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e}continue}- if err := dedupeReporter.UnitOk(ctx, RepoUnit{Name: name, URL: url}); err != nil {++ var installationID int64 = 0+ if ac, ok := s.connector.(*appConnector); ok {+ ac.ensureRepoInstallation(url, name)+ installationID, _ = ac.installationIDForRepo(url)+ }++ ru := RepoUnit{Name: name, URL: url, InstallationID: installationID}++ if err := dedupeReporter.UnitOk(ctx, ru); err != nil {return err}}@@ -2428,8 +2504,14 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporterepoURL, kind := unit.SourceUnitID()ctx = context.WithValue(ctx, "repo", repoURL)- if connector, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations && kind == "repo" {- if err := s.mapReposToInstallations(ctx, connector, []string{repoURL}); err != nil {+ // If the unit has an installation ID, use it. Fetching it is slow,+ // specifically it stalls large multi-org scans on rate limits (INT-790).++ // [CG] This is pretty ugly; if you can clean it up please do (I failed).+ if ac, ok := s.connector.(*appConnector); ok && s.conn.ScanAllInstallations && kind == "repo" {+ if ru, ok := unit.(RepoUnit); ok && ru.InstallationID > 0 {+ ac.setRepoInstallationForRepoName(repoURL, ru.Name, ru.InstallationID)+ } else if err := s.mapReposToInstallations(ctx, ac, []string{repoURL}); err != nil {return err}}diff --git a/pkg/sources/github/map_installations_test.go b/pkg/sources/github/map_installations_test.gonew file mode 100644index 000000000..51c4fda51--- /dev/null+++ b/pkg/sources/github/map_installations_test.go@@ -0,0 +1,167 @@+package github++import (+ "encoding/json"+ "testing"++ "github.com/google/go-github/v67/github"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++// This is meant to track APIUnitReporter.UnitOK in scanner/cmd/scanner/pipeline.go+// precisely+func newMarshalledRepoUnit(t *testing.T, su sources.SourceUnit) []byte {+ t.Helper()++ id, kind := su.SourceUnitID()++ unitData, err := json.Marshal(su)+ require.NoError(t, err)++ out, err := json.Marshal(map[string]any{+ "id": id,+ "kind": string(kind),+ "display": su.Display(),+ "unit_data": unitData,+ })+ require.NoError(t, err)++ return out+}++func TestGitHub_UnmarshalSourceUnit(t *testing.T) {+ s := &Source{}++ t.Run("source unit with unit_data recovers installation id", func(t *testing.T) {+ data := newMarshalledRepoUnit(t, RepoUnit{+ Name: "spotify/backstage",+ URL: "https://github.com/spotify/backstage.git",+ InstallationID: 2448,+ })++ unit, err := s.UnmarshalSourceUnit(data)+ require.NoError(t, err)++ repoUnit, ok := unit.(RepoUnit)+ require.True(t, ok, "expected a RepoUnit so ChunkUnit can read the installation id")+ assert.Equal(t, "https://github.com/spotify/backstage.git", repoUnit.URL)+ assert.Equal(t, "spotify/backstage", repoUnit.Name)+ assert.Equal(t, int64(2448), repoUnit.InstallationID)++ id, kind := unit.SourceUnitID()+ assert.Equal(t, "https://github.com/spotify/backstage.git", id)+ assert.EqualValues(t, "repo", kind)+ })++ t.Run("legacy source unit without unit_data has no installation id", func(t *testing.T) {+ data, err := json.Marshal(map[string]any{+ "id": "https://github.com/acme/widgets.git",+ "kind": "repo",+ "display": "acme/widgets",+ })+ require.NoError(t, err)++ unit, err := s.UnmarshalSourceUnit(data)+ require.NoError(t, err)++ repoUnit, ok := unit.(RepoUnit)+ require.True(t, ok)+ assert.Equal(t, "https://github.com/acme/widgets.git", repoUnit.URL)+ assert.Equal(t, "acme/widgets", repoUnit.Name)+ assert.Zero(t, repoUnit.InstallationID, "legacy units fall back to deriving the installation")+ })++ t.Run("full enumeration payload", func(t *testing.T) {+ data, err := json.Marshal(RepoUnit{+ Name: "acme/widgets",+ URL: "https://github.com/acme/widgets.git",+ InstallationID: 111,+ })+ require.NoError(t, err)++ unit, err := s.UnmarshalSourceUnit(data)+ require.NoError(t, err)++ repoUnit, ok := unit.(RepoUnit)+ require.True(t, ok)+ assert.Equal(t, int64(111), repoUnit.InstallationID)+ })++ t.Run("gist unit round trips", func(t *testing.T) {+ data := newMarshalledRepoUnit(t, GistUnit{Name: "abc", URL: "https://gist.github.com/abc.git"})++ unit, err := s.UnmarshalSourceUnit(data)+ require.NoError(t, err)++ _, ok := unit.(GistUnit)+ require.True(t, ok)+ _, kind := unit.SourceUnitID()+ assert.EqualValues(t, "gist", kind)+ })++ t.Run("rejects unrecognized payload", func(t *testing.T) {+ _, err := s.UnmarshalSourceUnit([]byte(`{"unrelated":"value"}`))+ require.Error(t, err)+ })+}++// Ensure we store installation IDs when enumerating+func TestGitHub_ProcessReposEmitsInstallationID(t *testing.T) {+ ctx := context.Background()++ connector := &appConnector{+ installationID: 1,+ repoInstallationMap: make(map[string]int64),+ }+ s := &Source{+ conn: &sourcespb.GitHub{ScanAllInstallations: true},+ connector: connector,+ filteredRepoCache: &filteredRepoCache{Cache: simple.NewCache[string]()},+ repoInfoCache: newRepoInfoCache(),+ }++ const installID int64 = 2448+ repo := &github.Repository{+ Name: github.String("backstage"),+ FullName: github.String("spotify/backstage"),+ CloneURL: github.String("https://github.com/spotify/backstage.git"),+ Owner: &github.User{Login: github.String("spotify"), Type: github.String("Organization")},+ }++ // Mirror enumerateAllInstallationRepos: the lister records the installation+ // for each repo in a page before processRepos emits the units for it.+ page := 0+ listRepos := func(ctx context.Context, _ string, _ repoListOptions) ([]*github.Repository, *github.Response, error) {+ page+++ if page > 1 {+ return nil, &github.Response{}, nil+ }+ connector.setRepoInstallationForRepoName(repo.GetCloneURL(), repo.GetName(), installID)+ return []*github.Repository{repo}, &github.Response{}, nil+ }++ var units []sources.SourceUnit+ reporter := sources.VisitorReporter{+ VisitUnit: func(_ context.Context, u sources.SourceUnit) error {+ units = append(units, u)+ return nil+ },+ }++ err := s.processRepos(ctx, "spotify", reporter, listRepos, &appListOptions{+ ListOptions: github.ListOptions{PerPage: 100},+ })+ require.NoError(t, err)++ require.Len(t, units, 1)+ repoUnit, ok := units[0].(RepoUnit)+ require.True(t, ok)+ assert.Equal(t, "https://github.com/spotify/backstage.git", repoUnit.URL)+ assert.Equal(t, installID, repoUnit.InstallationID)+}diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.goindex 6a169233a..5ac878a8c 100644--- a/pkg/sources/github/repo.go+++ b/pkg/sources/github/repo.go@@ -300,14 +300,22 @@ func (s *Source) processRepos(ctx context.Context, target string, reporter sourcs.totalRepoSize += r.GetSize()s.filteredRepoCache.Set(repoName, repoURL)s.cacheRepoInfo(r)++ unit := RepoUnit{Name: repoName, URL: repoURL}// Repos enumerated outside installation listings (e.g. member// personal repos) belong to no installation; map them to the// default installation so the scan-all mapping in ChunkUnit// doesn't reject repos we just enumerated.if connector, ok := s.connector.(*appConnector); ok && s.conn.GetScanAllInstallations() {connector.ensureRepoInstallation(repoURL, r.GetName())+ // Record the resolved installation on the unit so ChunkUnit+ // scans it with the correct token without re-listing every+ // installation's repos (INT-790).+ if installID, mapped := connector.installationIDForRepo(repoURL); mapped {+ unit.InstallationID = installID+ }}- if err := reporter.UnitOk(ctx, RepoUnit{Name: repoName, URL: repoURL}); err != nil {+ if err := reporter.UnitOk(ctx, unit); err != nil {return err}