Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add Circle CI source (#997)
Dustin Decker5f6143f09aa3
diff --git a/README.md b/README.mdindex d2639a6f6..4614dc4e8 100644--- a/README.md+++ b/README.md@@ -96,7 +96,7 @@ trufflehog s3 --bucket=<bucket name> --only-verifiedTruffleHog v3 is a complete rewrite in Go with many new powerful features.- We've **added over 700 credential detectors that support active verification against their respective APIs**.-- We've also added native **support for scanning GitHub, GitLab, filesystems, and S3**.+- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, and Circle CI**.- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.@@ -160,6 +160,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:- S3- filesystem- syslog+- circleci- file and stdin (coming soon)Each subcommand can have options that you can see with the `--help` flag provided to the sub command:diff --git a/main.go b/main.goindex f6d77ece7..e8802abfd 100644--- a/main.go+++ b/main.go@@ -95,6 +95,9 @@ var (syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String()syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String()syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").String()++ circleCiScan = cli.Command("circleci", "Scan CircleCI")+ circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String())func init() {@@ -293,6 +296,10 @@ func run(state overseer.State) {if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {logrus.WithError(err).Fatal("Failed to scan syslog.")}+ case circleCiScan.FullCommand():+ if err = e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {+ logrus.WithError(err).Fatal("Failed to scan CircleCI.")+ }}// asynchronously wait for scanning to finish and cleanupgo e.Finish(ctx)diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.gonew file mode 100644index 000000000..7c3625f8f--- /dev/null+++ b/pkg/engine/circleci.go@@ -0,0 +1,48 @@+package engine++import (+ "runtime"++ "github.com/go-errors/errors"+ "github.com/sirupsen/logrus"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"+)++// ScanS3 scans CircleCI logs.+func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {+ connection := &sourcespb.CircleCI{+ Credential: &sourcespb.CircleCI_Token{+ Token: token,+ },+ }++ var conn anypb.Any+ err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})+ if err != nil {+ logrus.WithError(err).Error("failed to marshal Circle CI connection")+ return err+ }++ circleSource := circleci.Source{}+ err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())+ if err != nil {+ return errors.WrapPrefix(err, "failed to init Circle CI source", 0)+ }++ e.sourcesWg.Add(1)+ go func() {+ defer common.RecoverWithExit(ctx)+ defer e.sourcesWg.Done()+ err := circleSource.Chunks(ctx, e.ChunksChan())+ if err != nil {+ logrus.WithError(err).Error("error scanning Circle CI")+ }+ }()+ return nil+}diff --git a/pkg/engine/s3.go b/pkg/engine/s3.goindex 98f9ca721..900670e1f 100644--- a/pkg/engine/s3.go+++ b/pkg/engine/s3.go@@ -42,7 +42,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {var conn anypb.Anyerr := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})if err != nil {- logrus.WithError(err).Error("failed to marshal github connection")+ logrus.WithError(err).Error("failed to marshal S3 connection")return err}@@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {defer e.sourcesWg.Done()err := s3Source.Chunks(ctx, e.ChunksChan())if err != nil {- logrus.WithError(err).Error("error scanning s3")+ logrus.WithError(err).Error("error scanning S3")}}()return nildiff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.gonew file mode 100644index 000000000..7366b9076--- /dev/null+++ b/pkg/sources/circleci/circleci.go@@ -0,0 +1,256 @@+package circleci++import (+ "bytes"+ "encoding/json"+ "fmt"+ "io/ioutil"+ "net/http"++ "github.com/go-errors/errors"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/types/known/anypb"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++const (+ baseURL = "https://circleci.com/api/v1.1/"+)++type Source struct {+ name string+ sourceId int64+ jobId int64+ verify bool+ sources.Progress+ token string+ client *http.Client+}++// Ensure the Source satisfies the interface at compile time+var _ sources.Source = (*Source)(nil)++// Type returns the type of source.+// It is used for matching source types in configuration and job input.+func (s *Source) Type() sourcespb.SourceType {+ return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI+}++func (s *Source) SourceID() int64 {+ return s.sourceId+}++func (s *Source) JobID() int64 {+ return s.jobId+}++// Init returns an initialized Filesystem source.+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {+ s.name = name+ s.sourceId = sourceId+ s.jobId = jobId+ s.verify = verify+ s.client = common.RetryableHttpClientTimeout(3)++ var conn sourcespb.CircleCI+ if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {+ return errors.WrapPrefix(err, "error unmarshalling connection", 0)+ }++ switch conn.Credential.(type) {+ case *sourcespb.CircleCI_Token:+ s.token = conn.GetToken()+ }++ return nil+}++// Chunks emits chunks of bytes over a channel.+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+ projects, err := s.projects(ctx)+ if err != nil {+ return err+ }++ for _, proj := range projects {+ builds, err := s.buildsForProject(ctx, proj)+ if err != nil {+ return err+ }++ for _, bld := range builds {+ buildSteps, err := s.stepsForBuild(ctx, proj, bld)+ if err != nil {+ return err+ }++ for _, step := range buildSteps {+ for _, action := range step.Actions {+ err = s.chunkAction(ctx, proj, bld, action, step.Name, chunksChan)+ if err != nil {+ return err+ }+ }+ }+ }+ }++ return nil+}++type project struct {+ VCS string `json:"vcs_type"`+ Username string `json:"username"`+ RepoName string `json:"reponame"`+}++func (s *Source) projects(ctx context.Context) ([]project, error) {+ reqURL := fmt.Sprintf("%sprojects", baseURL)+ req, err := http.NewRequest("GET", reqURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Circle-Token", s.token)+ req.Header.Set("Accept", "application/json")+ res, err := s.client.Do(req)+ if err != nil {+ return nil, err+ }+ defer res.Body.Close()++ if res.StatusCode > 399 && res.StatusCode < 500 {+ return nil, fmt.Errorf("invalid credentials, status %d", res.StatusCode)+ }++ var projects []project+ if err := json.NewDecoder(res.Body).Decode(&projects); err != nil {+ return nil, err+ }++ return projects, nil+}++type build struct {+ BuildNum int `json:"build_num"`+}++func (s *Source) buildsForProject(ctx context.Context, proj project) ([]build, error) {+ reqURL := fmt.Sprintf("%sproject/%s/%s/%s", baseURL, proj.VCS, proj.Username, proj.RepoName)+ req, err := http.NewRequest("GET", reqURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Circle-Token", s.token)+ req.Header.Set("Accept", "application/json")+ res, err := s.client.Do(req)+ if err != nil {+ return nil, err+ }+ defer res.Body.Close()++ var builds []build+ if err := json.NewDecoder(res.Body).Decode(&builds); err != nil {+ return nil, err+ }++ return builds, nil+}++type action struct {+ Index int `json:"index"`+ OutputURL string `json:"output_url"`+}++type buildStep struct {+ Name string `json:"name"`+ Actions []action `json:"actions"`+}++func (s *Source) stepsForBuild(ctx context.Context, proj project, bld build) ([]buildStep, error) {+ reqURL := fmt.Sprintf("%sproject/%s/%s/%s/%d", baseURL, proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)+ req, err := http.NewRequest("GET", reqURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Circle-Token", s.token)+ req.Header.Set("Accept", "application/json")+ res, err := s.client.Do(req)+ if err != nil {+ return nil, err+ }+ defer res.Body.Close()++ type buildRes struct {+ Steps []buildStep `json:"steps"`+ }++ var bldRes buildRes+ if err := json.NewDecoder(res.Body).Decode(&bldRes); err != nil {+ return nil, err+ }++ return bldRes.Steps, nil+}++func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {+ req, err := http.NewRequest("GET", act.OutputURL, nil)+ if err != nil {+ return err+ }+ res, err := s.client.Do(req)+ if err != nil {+ return err+ }+ defer res.Body.Close()+ logOutput, err := ioutil.ReadAll(res.Body)+ if err != nil {+ return err+ }++ linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)++ chunk := &sources.Chunk{+ SourceType: s.Type(),+ SourceName: s.name,+ SourceID: s.SourceID(),+ Data: removeCircleSha1Line(logOutput),+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Circleci{+ Circleci: &source_metadatapb.CircleCI{+ VcsType: proj.VCS,+ Username: proj.Username,+ Repository: proj.RepoName,+ BuildNumber: int64(bld.BuildNum),+ BuildStep: stepName,+ Link: linkURL,+ },+ },+ },+ Verify: s.verify,+ }++ chunksChan <- chunk++ return nil+}++func removeCircleSha1Line(input []byte) []byte {+ // Split the input slice into a slice of lines+ lines := bytes.Split(input, []byte("\n"))++ // Iterate over the lines and add the ones that don't contain "CIRCLE_SHA1=" to the result slice+ result := make([][]byte, 0, len(lines))+ for _, line := range lines {+ if !bytes.Contains(line, []byte("CIRCLE_SHA1=")) {+ result = append(result, line)+ }+ }++ // Join the lines in the result slice and return the resulting slice+ return bytes.Join(result, []byte("\n"))+}diff --git a/pkg/sources/circleci/circleci_test.go b/pkg/sources/circleci/circleci_test.gonew file mode 100644index 000000000..2be4891f0--- /dev/null+++ b/pkg/sources/circleci/circleci_test.go@@ -0,0 +1,93 @@+package circleci++import (+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "google.golang.org/protobuf/types/known/anypb"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"+)++func TestSource_Scan(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)+ defer cancel()++ secret, err := common.GetTestSecret(ctx)+ if err != nil {+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))+ }+ token := secret.MustGetField("CIRCLECI_TOKEN")++ type init struct {+ name string+ verify bool+ connection *sourcespb.CircleCI+ }+ tests := []struct {+ name string+ init init+ wantSourceMetadata *source_metadatapb.MetaData+ wantErr bool+ }{+ {+ name: "get a chunk",+ init: init{+ name: "example repo",+ connection: &sourcespb.CircleCI{+ Credential: &sourcespb.CircleCI_Token{+ Token: token,+ },+ },+ verify: true,+ },+ wantSourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Circleci{+ Circleci: &source_metadatapb.CircleCI{+ VcsType: "github",+ Username: "dustin-decker",+ Repository: "circle-ci",+ BuildNumber: 2,+ BuildStep: "Spin up environment",+ Link: "https://app.circleci.com/pipelines/github/dustin-decker/circle-ci/2",+ },+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Source{}++ conn, err := anypb.New(tt.init.connection)+ if err != nil {+ t.Fatal(err)+ }++ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)+ if (err != nil) != tt.wantErr {+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ chunksCh := make(chan *sources.Chunk, 1)+ go func() {+ err = s.Chunks(ctx, chunksCh)+ if (err != nil) != tt.wantErr {+ t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ }()+ gotChunk := <-chunksCh+ if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {+ t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}