Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Automatically update trufflehog (#121)
Bill Richa8f07c8e97ba
diff --git a/.goreleaser.yml b/.goreleaser.ymlindex 2ba53f5a6..6ca48616e 100644--- a/.goreleaser.yml+++ b/.goreleaser.yml@@ -1,4 +1,6 @@builds:+ ldflags:+ - -X 'github.com/trufflesecurity/trufflehog/v3/pkg/version.BuildVersion={{ .Version }}'- env: [CGO_ENABLED=0]binary: trufflehoggoos:@@ -42,4 +44,4 @@ docker_manifests:- name_template: ghcr.io/trufflesecurity/{{ .ProjectName }}:latestimage_templates:- ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64- - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8\ No newline at end of file+ - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8diff --git a/go.mod b/go.modindex e3b36dcca..a19fc49f8 100644--- a/go.mod+++ b/go.mod@@ -4,6 +4,8 @@ go 1.17replace github.com/gitleaks/go-gitdiff => github.com/bill-rich/go-gitdiff v0.7.6-custom1+replace github.com/jpillora/overseer => github.com/dustin-decker/overseer v1.1.7-custom2+require (cloud.google.com/go/secretmanager v1.3.0github.com/Azure/go-autorest/autorest/azure/auth v0.5.11@@ -13,6 +15,7 @@ require (github.com/aws/aws-sdk-go-v2/service/sts v1.16.2github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fbgithub.com/bradleyfalzon/ghinstallation/v2 v2.0.4+ github.com/denisbrodbeck/machineid v1.0.1github.com/envoyproxy/protoc-gen-validate v0.6.7github.com/fatih/color v1.13.0github.com/felixge/fgprof v0.9.2@@ -25,6 +28,7 @@ require (github.com/h2non/filetype v1.1.3github.com/hashicorp/go-retryablehttp v0.7.0github.com/joho/godotenv v1.4.0+ github.com/jpillora/overseer v1.1.6github.com/kylelemons/godebug v1.1.0github.com/mattn/go-colorable v0.1.12github.com/paulbellamy/ratecounter v0.2.0@@ -70,6 +74,7 @@ require (github.com/emirpasic/gods v1.12.0 // indirectgithub.com/go-git/gcfg v1.5.0 // indirectgithub.com/go-git/go-billy/v5 v5.3.1 // indirect+ github.com/go-ole/go-ole v1.2.6 // indirectgithub.com/golang-jwt/jwt/v4 v4.2.0 // indirectgithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirectgithub.com/golang/protobuf v1.5.2 // indirect@@ -82,6 +87,7 @@ require (github.com/imdario/mergo v0.3.12 // indirectgithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirectgithub.com/jmespath/go-jmespath v0.4.0 // indirect+ github.com/jpillora/s3 v1.1.4 // indirectgithub.com/json-iterator/go v1.1.11 // indirectgithub.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirectgithub.com/mattn/go-isatty v0.0.14 // indirect@@ -90,6 +96,7 @@ require (github.com/modern-go/reflect2 v1.0.1 // indirectgithub.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirectgithub.com/xanzy/ssh-agent v0.3.0 // indirect+ github.com/yusufpapurcu/wmi v1.2.2 // indirectgo.opencensus.io v0.23.0 // indirectgolang.org/x/mod v0.5.0 // indirectgolang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirectdiff --git a/go.sum b/go.sumindex 7eb8df887..4837610de 100644--- a/go.sum+++ b/go.sum@@ -146,8 +146,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=+github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=+github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=+github.com/dustin-decker/overseer v1.1.7-custom2 h1:k6lSAUkdxnICmN5/sEsCKFVYchv2wBfogAdwONk39y0=+github.com/dustin-decker/overseer v1.1.7-custom2/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=@@ -186,6 +190,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+ztigithub.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=@@ -331,6 +337,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwgithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=+github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=+github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=@@ -416,7 +424,11 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBgithub.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=+github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=+github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=+github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=+github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=@@ -447,6 +459,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9degithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=github.com/zricethezav/gitleaks/v8 v8.5.2 h1:+IiLvmBiMGetpgNtzC71EHktTXc1Zt7m9AwnqhhJ04g=github.com/zricethezav/gitleaks/v8 v8.5.2/go.mod h1:TxBuxH8eB/1la8Mc7I6j/ZZYNG/mHGpoi09N2oHx5nQ=go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=diff --git a/main.go b/main.goindex 567422efb..0fa991645 100644--- a/main.go+++ b/main.go@@ -4,7 +4,6 @@ import ("context""encoding/json""fmt"- "gopkg.in/alecthomas/kingpin.v2""log""net/http"_ "net/http/pprof"@@ -12,8 +11,14 @@ import ("runtime""strconv""strings"+ "syscall""time"+ "github.com/jpillora/overseer"+ "github.com/trufflesecurity/trufflehog/v3/pkg/updater"+ "github.com/trufflesecurity/trufflehog/v3/pkg/version"+ "gopkg.in/alecthomas/kingpin.v2"+"github.com/felixge/fgprof""github.com/gorilla/mux"@@ -25,9 +30,91 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources/git")+var (+ cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")+ cmd string+ debug = cli.Flag("debug", "Run in debug mode").Bool()+ versionFlag = cli.Flag("version", "Prints trufflehog version.").Bool()+ jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()+ jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()+ concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()+ noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()+ onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()+ // rules = cli.Flag("rules", "Path to file with custom rules.").String()+ printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()+ noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()++ gitScan = cli.Command("git", "Find credentials in git repositories.")+ gitScanURI = gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()+ gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()+ gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()+ gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()+ gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()+ gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()+ _ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()+ _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()+ _ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()++ githubScan = cli.Command("github", "Find credentials in GitHub repositories.")+ githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()+ githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()+ githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()+ githubScanToken = githubScan.Flag("token", "GitHub token.").String()+ githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()+ githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()++ gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")+ // TODO: Add more GitLab options+ gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()+ gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()+ gitlabScanToken = gitlabScan.Flag("token", "GitLab token.").Required().String()++ filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")+ filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()+ // TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.+ // filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()+ // filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()+ // filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()++ s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")+ s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate.").String()+ s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate.").String()+ s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()+ s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()+)++func init() {+ for i, arg := range os.Args {+ if strings.HasPrefix(arg, "--") {+ os.Args[i] = strings.ReplaceAll(arg, "_", "-")+ }+ }++ cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))+}+func main() {+ updateCfg := overseer.Config{+ Program: run,+ Debug: *debug,+ RestartSignal: syscall.SIGTERM,+ // TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15+ // PreUpgrade: checkUpdateSignature(binaryPath string),+ }++ if !*noUpdate {+ updateCfg.Fetcher = updater.Fetcher(version.BuildVersion)+ }+ err := overseer.RunErr(updateCfg)+ if err != nil {+ logrus.WithError(err).Fatal("error occured with trufflehog updater 🐷")+ }+}++func run(state overseer.State) {cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")debug := cli.Flag("debug", "Run in debug mode").Bool()+ versionFlag := cli.Flag("version", "Prints trufflehog version.").Bool()jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool()jsonLegacy := cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()@@ -82,6 +169,11 @@ func main() {cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))+ if *versionFlag {+ fmt.Println("trufflehog " + version.BuildVersion)+ return+ }+// When setting a base commit, chunks must be scanned in order.if *gitScanSinceCommit != "" {*concurrency = 1diff --git a/pkg/updater/updater.go b/pkg/updater/updater.gonew file mode 100644index 000000000..c41face4f--- /dev/null+++ b/pkg/updater/updater.go@@ -0,0 +1,112 @@+package updater++import (+ "archive/tar"+ "archive/zip"+ "bytes"+ "compress/gzip"+ "encoding/json"+ "io"+ "io/ioutil"+ "net/http"+ "runtime"+ "strings"+ "time"++ "github.com/go-errors/errors"+ "github.com/jpillora/overseer/fetcher"+ log "github.com/sirupsen/logrus"++ "github.com/trufflesecurity/trufflehog/v3/pkg/version"+)++func Fetcher(version string) fetcher.Interface {+ return &OSS{+ CurrentVersion: version,+ }+}++type OSS struct {+ Interval time.Duration+ CurrentVersion string+}++// Init validates the provided config+func (g *OSS) Init() error {+ //initiate OSS connection+ return nil+}++const url = "https://oss.trufflehog.org/updates"++type FormData struct {+ OS string+ Arch string+ CurrentVersion string+ Timezone string+ Binary string+}++// Fetch binary from URL via OSS client+func (g *OSS) Fetch() (io.Reader, error) {+ log.Debug("fetching trufflehog update")++ zone, _ := time.Now().Zone()+ data := &FormData{+ OS: runtime.GOOS,+ Arch: runtime.GOARCH,+ CurrentVersion: version.BuildVersion,+ Timezone: zone,+ Binary: "trufflehog",+ }++ dataByte, err := json.Marshal(data)+ if err != nil {+ return nil, err+ }+ reader := bytes.NewReader(dataByte)+ resp, err := http.Post(url, "application/json", reader)+ if err != nil || resp == nil {+ return nil, err+ }+ defer resp.Body.Close()++ newBinBytes, err := ioutil.ReadAll(resp.Body)+ if err != nil {+ return nil, err+ }++ buffer := bytes.NewReader(newBinBytes)+ switch runtime.GOOS {+ case "windows":+ zipReader, err := zip.NewReader(buffer, int64(len(newBinBytes)))+ if err != nil {+ return nil, errors.Errorf("Failed to read zip archive: %s", err)+ }+ for _, f := range zipReader.File {+ if strings.HasPrefix(f.Name, "trufflehog") {+ return f.Open()+ }+ }+ default:+ gzipReader, err := gzip.NewReader(buffer)+ if err != nil {+ return nil, errors.Errorf("Failed to read gzip archive: %s", err)+ }+ defer gzipReader.Close()+ tarReader := tar.NewReader(gzipReader)+ for {+ header, err := tarReader.Next()+ if err == io.EOF {+ return nil, errors.New("unable to get update")+ }++ if header.Typeflag == tar.TypeReg {+ if strings.HasPrefix(header.Name, "trufflehog") {+ return tarReader, nil+ }+ }+ }+ }+ return nil, errors.New("unable to get update")+}diff --git a/pkg/version/version.go b/pkg/version/version.gonew file mode 100644index 000000000..3c0765e71--- /dev/null+++ b/pkg/version/version.go@@ -0,0 +1,3 @@+package version++var BuildVersion = "dev"