Skip to content

Find, verify, and analyze leaked credentials

Add Circle CI source (#997)

Dustin Decker5f6143f09aa3
diff --git a/README.md b/README.md
index d2639a6f6..4614dc4e8 100644
--- a/README.md
+++ b/README.md
@@ -96,7 +96,7 @@ trufflehog s3 --bucket=<bucket name> --only-verified
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
- We've **added over 700 credential detectors that support active verification against their respective APIs**.
-- We've also added native **support for scanning GitHub, GitLab, filesystems, and S3**.
+- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, and Circle CI**.
- **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology.
@@ -160,6 +160,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
- S3
- filesystem
- syslog
+- circleci
- file and stdin (coming soon)
Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
diff --git a/main.go b/main.go
index f6d77ece7..e8802abfd 100644
--- a/main.go
+++ b/main.go
@@ -95,6 +95,9 @@ var (
syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String()
syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String()
syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").String()
+
+ circleCiScan = cli.Command("circleci", "Scan CircleCI")
+ circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
)
func init() {
@@ -293,6 +296,10 @@ func run(state overseer.State) {
if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
logrus.WithError(err).Fatal("Failed to scan syslog.")
}
+ case circleCiScan.FullCommand():
+ if err = e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
+ logrus.WithError(err).Fatal("Failed to scan CircleCI.")
+ }
}
// asynchronously wait for scanning to finish and cleanup
go e.Finish(ctx)
diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go
new file mode 100644
index 000000000..7c3625f8f
--- /dev/null
+++ b/pkg/engine/circleci.go
@@ -0,0 +1,48 @@
+package engine
+
+import (
+ "runtime"
+
+ "github.com/go-errors/errors"
+ "github.com/sirupsen/logrus"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
+)
+
+// ScanS3 scans CircleCI logs.
+func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
+ connection := &sourcespb.CircleCI{
+ Credential: &sourcespb.CircleCI_Token{
+ Token: token,
+ },
+ }
+
+ var conn anypb.Any
+ err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
+ if err != nil {
+ logrus.WithError(err).Error("failed to marshal Circle CI connection")
+ return err
+ }
+
+ circleSource := circleci.Source{}
+ err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
+ if err != nil {
+ return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
+ }
+
+ e.sourcesWg.Add(1)
+ go func() {
+ defer common.RecoverWithExit(ctx)
+ defer e.sourcesWg.Done()
+ err := circleSource.Chunks(ctx, e.ChunksChan())
+ if err != nil {
+ logrus.WithError(err).Error("error scanning Circle CI")
+ }
+ }()
+ return nil
+}
diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go
index 98f9ca721..900670e1f 100644
--- a/pkg/engine/s3.go
+++ b/pkg/engine/s3.go
@@ -42,7 +42,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
- logrus.WithError(err).Error("failed to marshal github connection")
+ logrus.WithError(err).Error("failed to marshal S3 connection")
return err
}
@@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
defer e.sourcesWg.Done()
err := s3Source.Chunks(ctx, e.ChunksChan())
if err != nil {
- logrus.WithError(err).Error("error scanning s3")
+ logrus.WithError(err).Error("error scanning S3")
}
}()
return nil
diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go
new file mode 100644
index 000000000..7366b9076
--- /dev/null
+++ b/pkg/sources/circleci/circleci.go
@@ -0,0 +1,256 @@
+package circleci
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+
+ "github.com/go-errors/errors"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const (
+ baseURL = "https://circleci.com/api/v1.1/"
+)
+
+type Source struct {
+ name string
+ sourceId int64
+ jobId int64
+ verify bool
+ sources.Progress
+ token string
+ client *http.Client
+}
+
+// Ensure the Source satisfies the interface at compile time
+var _ sources.Source = (*Source)(nil)
+
+// Type returns the type of source.
+// It is used for matching source types in configuration and job input.
+func (s *Source) Type() sourcespb.SourceType {
+ return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
+}
+
+func (s *Source) SourceID() int64 {
+ return s.sourceId
+}
+
+func (s *Source) JobID() int64 {
+ return s.jobId
+}
+
+// Init returns an initialized Filesystem source.
+func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
+ s.name = name
+ s.sourceId = sourceId
+ s.jobId = jobId
+ s.verify = verify
+ s.client = common.RetryableHttpClientTimeout(3)
+
+ var conn sourcespb.CircleCI
+ if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+ return errors.WrapPrefix(err, "error unmarshalling connection", 0)
+ }
+
+ switch conn.Credential.(type) {
+ case *sourcespb.CircleCI_Token:
+ s.token = conn.GetToken()
+ }
+
+ return nil
+}
+
+// Chunks emits chunks of bytes over a channel.
+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
+ projects, err := s.projects(ctx)
+ if err != nil {
+ return err
+ }
+
+ for _, proj := range projects {
+ builds, err := s.buildsForProject(ctx, proj)
+ if err != nil {
+ return err
+ }
+
+ for _, bld := range builds {
+ buildSteps, err := s.stepsForBuild(ctx, proj, bld)
+ if err != nil {
+ return err
+ }
+
+ for _, step := range buildSteps {
+ for _, action := range step.Actions {
+ err = s.chunkAction(ctx, proj, bld, action, step.Name, chunksChan)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+ }
+
+ return nil
+}
+
+type project struct {
+ VCS string `json:"vcs_type"`
+ Username string `json:"username"`
+ RepoName string `json:"reponame"`
+}
+
+func (s *Source) projects(ctx context.Context) ([]project, error) {
+ reqURL := fmt.Sprintf("%sprojects", baseURL)
+ req, err := http.NewRequest("GET", reqURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Circle-Token", s.token)
+ req.Header.Set("Accept", "application/json")
+ res, err := s.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+
+ if res.StatusCode > 399 && res.StatusCode < 500 {
+ return nil, fmt.Errorf("invalid credentials, status %d", res.StatusCode)
+ }
+
+ var projects []project
+ if err := json.NewDecoder(res.Body).Decode(&projects); err != nil {
+ return nil, err
+ }
+
+ return projects, nil
+}
+
+type build struct {
+ BuildNum int `json:"build_num"`
+}
+
+func (s *Source) buildsForProject(ctx context.Context, proj project) ([]build, error) {
+ reqURL := fmt.Sprintf("%sproject/%s/%s/%s", baseURL, proj.VCS, proj.Username, proj.RepoName)
+ req, err := http.NewRequest("GET", reqURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Circle-Token", s.token)
+ req.Header.Set("Accept", "application/json")
+ res, err := s.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+
+ var builds []build
+ if err := json.NewDecoder(res.Body).Decode(&builds); err != nil {
+ return nil, err
+ }
+
+ return builds, nil
+}
+
+type action struct {
+ Index int `json:"index"`
+ OutputURL string `json:"output_url"`
+}
+
+type buildStep struct {
+ Name string `json:"name"`
+ Actions []action `json:"actions"`
+}
+
+func (s *Source) stepsForBuild(ctx context.Context, proj project, bld build) ([]buildStep, error) {
+ reqURL := fmt.Sprintf("%sproject/%s/%s/%s/%d", baseURL, proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
+ req, err := http.NewRequest("GET", reqURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Circle-Token", s.token)
+ req.Header.Set("Accept", "application/json")
+ res, err := s.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+
+ type buildRes struct {
+ Steps []buildStep `json:"steps"`
+ }
+
+ var bldRes buildRes
+ if err := json.NewDecoder(res.Body).Decode(&bldRes); err != nil {
+ return nil, err
+ }
+
+ return bldRes.Steps, nil
+}
+
+func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error {
+ req, err := http.NewRequest("GET", act.OutputURL, nil)
+ if err != nil {
+ return err
+ }
+ res, err := s.client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer res.Body.Close()
+ logOutput, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return err
+ }
+
+ linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum)
+
+ chunk := &sources.Chunk{
+ SourceType: s.Type(),
+ SourceName: s.name,
+ SourceID: s.SourceID(),
+ Data: removeCircleSha1Line(logOutput),
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Circleci{
+ Circleci: &source_metadatapb.CircleCI{
+ VcsType: proj.VCS,
+ Username: proj.Username,
+ Repository: proj.RepoName,
+ BuildNumber: int64(bld.BuildNum),
+ BuildStep: stepName,
+ Link: linkURL,
+ },
+ },
+ },
+ Verify: s.verify,
+ }
+
+ chunksChan <- chunk
+
+ return nil
+}
+
+func removeCircleSha1Line(input []byte) []byte {
+ // Split the input slice into a slice of lines
+ lines := bytes.Split(input, []byte("\n"))
+
+ // Iterate over the lines and add the ones that don't contain "CIRCLE_SHA1=" to the result slice
+ result := make([][]byte, 0, len(lines))
+ for _, line := range lines {
+ if !bytes.Contains(line, []byte("CIRCLE_SHA1=")) {
+ result = append(result, line)
+ }
+ }
+
+ // Join the lines in the result slice and return the resulting slice
+ return bytes.Join(result, []byte("\n"))
+}
diff --git a/pkg/sources/circleci/circleci_test.go b/pkg/sources/circleci/circleci_test.go
new file mode 100644
index 000000000..2be4891f0
--- /dev/null
+++ b/pkg/sources/circleci/circleci_test.go
@@ -0,0 +1,93 @@
+package circleci
+
+import (
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestSource_Scan(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
+ defer cancel()
+
+ secret, err := common.GetTestSecret(ctx)
+ if err != nil {
+ t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+ }
+ token := secret.MustGetField("CIRCLECI_TOKEN")
+
+ type init struct {
+ name string
+ verify bool
+ connection *sourcespb.CircleCI
+ }
+ tests := []struct {
+ name string
+ init init
+ wantSourceMetadata *source_metadatapb.MetaData
+ wantErr bool
+ }{
+ {
+ name: "get a chunk",
+ init: init{
+ name: "example repo",
+ connection: &sourcespb.CircleCI{
+ Credential: &sourcespb.CircleCI_Token{
+ Token: token,
+ },
+ },
+ verify: true,
+ },
+ wantSourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Circleci{
+ Circleci: &source_metadatapb.CircleCI{
+ VcsType: "github",
+ Username: "dustin-decker",
+ Repository: "circle-ci",
+ BuildNumber: 2,
+ BuildStep: "Spin up environment",
+ Link: "https://app.circleci.com/pipelines/github/dustin-decker/circle-ci/2",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Source{}
+
+ conn, err := anypb.New(tt.init.connection)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ chunksCh := make(chan *sources.Chunk, 1)
+ go func() {
+ err = s.Chunks(ctx, chunksCh)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ }()
+ gotChunk := <-chunksCh
+ if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" {
+ t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}