Skip to content

Find, verify, and analyze leaked credentials

Automatically update trufflehog (#121)

Bill Richa8f07c8e97ba
diff --git a/.goreleaser.yml b/.goreleaser.yml
index 2ba53f5a6..6ca48616e 100644
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -1,4 +1,6 @@
builds:
+ ldflags:
+ - -X 'github.com/trufflesecurity/trufflehog/v3/pkg/version.BuildVersion={{ .Version }}'
- env: [CGO_ENABLED=0]
binary: trufflehog
goos:
@@ -42,4 +44,4 @@ docker_manifests:
- name_template: ghcr.io/trufflesecurity/{{ .ProjectName }}:latest
image_templates:
- ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64
- - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8
\ No newline at end of file
+ - ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8
diff --git a/go.mod b/go.mod
index e3b36dcca..a19fc49f8 100644
--- a/go.mod
+++ b/go.mod
@@ -4,6 +4,8 @@ go 1.17
replace github.com/gitleaks/go-gitdiff => github.com/bill-rich/go-gitdiff v0.7.6-custom1
+replace github.com/jpillora/overseer => github.com/dustin-decker/overseer v1.1.7-custom2
+
require (
cloud.google.com/go/secretmanager v1.3.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
@@ -13,6 +15,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.16.2
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.0.4
+ github.com/denisbrodbeck/machineid v1.0.1
github.com/envoyproxy/protoc-gen-validate v0.6.7
github.com/fatih/color v1.13.0
github.com/felixge/fgprof v0.9.2
@@ -25,6 +28,7 @@ require (
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.0
github.com/joho/godotenv v1.4.0
+ github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
github.com/mattn/go-colorable v0.1.12
github.com/paulbellamy/ratecounter v0.2.0
@@ -70,6 +74,7 @@ require (
github.com/emirpasic/gods v1.12.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
+ github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang-jwt/jwt/v4 v4.2.0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
@@ -82,6 +87,7 @@ require (
github.com/imdario/mergo v0.3.12 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
+ github.com/jpillora/s3 v1.1.4 // indirect
github.com/json-iterator/go v1.1.11 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
@@ -90,6 +96,7 @@ require (
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
+ github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/mod v0.5.0 // indirect
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
diff --git a/go.sum b/go.sum
index 7eb8df887..4837610de 100644
--- a/go.sum
+++ b/go.sum
@@ -146,8 +146,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
+github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dustin-decker/overseer v1.1.7-custom2 h1:k6lSAUkdxnICmN5/sEsCKFVYchv2wBfogAdwONk39y0=
+github.com/dustin-decker/overseer v1.1.7-custom2/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -186,6 +190,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
@@ -331,6 +337,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
+github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -416,7 +424,11 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
+github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU=
+github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
@@ -447,6 +459,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
+github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zricethezav/gitleaks/v8 v8.5.2 h1:+IiLvmBiMGetpgNtzC71EHktTXc1Zt7m9AwnqhhJ04g=
github.com/zricethezav/gitleaks/v8 v8.5.2/go.mod h1:TxBuxH8eB/1la8Mc7I6j/ZZYNG/mHGpoi09N2oHx5nQ=
go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
diff --git a/main.go b/main.go
index 567422efb..0fa991645 100644
--- a/main.go
+++ b/main.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "gopkg.in/alecthomas/kingpin.v2"
"log"
"net/http"
_ "net/http/pprof"
@@ -12,8 +11,14 @@ import (
"runtime"
"strconv"
"strings"
+ "syscall"
"time"
+ "github.com/jpillora/overseer"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/updater"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/version"
+ "gopkg.in/alecthomas/kingpin.v2"
+
"github.com/felixge/fgprof"
"github.com/gorilla/mux"
@@ -25,9 +30,91 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
+var (
+ cli = kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
+ cmd string
+ debug = cli.Flag("debug", "Run in debug mode").Bool()
+ versionFlag = cli.Flag("version", "Prints trufflehog version.").Bool()
+ jsonOut = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
+ jsonLegacy = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
+ concurrency = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
+ noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
+ onlyVerified = cli.Flag("only-verified", "Only output verified results.").Bool()
+ // rules = cli.Flag("rules", "Path to file with custom rules.").String()
+ printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
+ noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
+
+ gitScan = cli.Command("git", "Find credentials in git repositories.")
+ gitScanURI = gitScan.Arg("uri", "Git repository URL. https:// or file:// schema expected.").Required().String()
+ gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+ gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+ gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String()
+ gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String()
+ gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
+ _ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
+ _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
+ _ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
+
+ githubScan = cli.Command("github", "Find credentials in GitHub repositories.")
+ githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+ githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+ githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+ githubScanToken = githubScan.Flag("token", "GitHub token.").String()
+ githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
+ githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+
+ gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
+ // TODO: Add more GitLab options
+ gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
+ gitlabScanRepos = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
+ gitlabScanToken = gitlabScan.Flag("token", "GitLab token.").Required().String()
+
+ filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
+ filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
+ // TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
+ // filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
+ // filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+ // filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+
+ s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
+ s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate.").String()
+ s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate.").String()
+ s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
+ s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
+)
+
+func init() {
+ for i, arg := range os.Args {
+ if strings.HasPrefix(arg, "--") {
+ os.Args[i] = strings.ReplaceAll(arg, "_", "-")
+ }
+ }
+
+ cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
+}
+
func main() {
+ updateCfg := overseer.Config{
+ Program: run,
+ Debug: *debug,
+ RestartSignal: syscall.SIGTERM,
+ // TODO: Eventually add a PreUpgrade func for signature check w/ x509 PKCS1v15
+ // PreUpgrade: checkUpdateSignature(binaryPath string),
+ }
+
+ if !*noUpdate {
+ updateCfg.Fetcher = updater.Fetcher(version.BuildVersion)
+ }
+ err := overseer.RunErr(updateCfg)
+ if err != nil {
+ logrus.WithError(err).Fatal("error occured with trufflehog updater 🐷")
+ }
+}
+
+func run(state overseer.State) {
cli := kingpin.New("TruffleHog", "TruffleHog is a tool for finding credentials.")
debug := cli.Flag("debug", "Run in debug mode").Bool()
+ versionFlag := cli.Flag("version", "Prints trufflehog version.").Bool()
jsonOut := cli.Flag("json", "Output in JSON format.").Short('j').Bool()
jsonLegacy := cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
concurrency := cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
@@ -82,6 +169,11 @@ func main() {
cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))
+ if *versionFlag {
+ fmt.Println("trufflehog " + version.BuildVersion)
+ return
+ }
+
// When setting a base commit, chunks must be scanned in order.
if *gitScanSinceCommit != "" {
*concurrency = 1
diff --git a/pkg/updater/updater.go b/pkg/updater/updater.go
new file mode 100644
index 000000000..c41face4f
--- /dev/null
+++ b/pkg/updater/updater.go
@@ -0,0 +1,112 @@
+package updater
+
+import (
+ "archive/tar"
+ "archive/zip"
+ "bytes"
+ "compress/gzip"
+ "encoding/json"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "runtime"
+ "strings"
+ "time"
+
+ "github.com/go-errors/errors"
+ "github.com/jpillora/overseer/fetcher"
+ log "github.com/sirupsen/logrus"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/version"
+)
+
+func Fetcher(version string) fetcher.Interface {
+ return &OSS{
+ CurrentVersion: version,
+ }
+}
+
+type OSS struct {
+ Interval time.Duration
+ CurrentVersion string
+}
+
+// Init validates the provided config
+func (g *OSS) Init() error {
+ //initiate OSS connection
+ return nil
+}
+
+const url = "https://oss.trufflehog.org/updates"
+
+type FormData struct {
+ OS string
+ Arch string
+ CurrentVersion string
+ Timezone string
+ Binary string
+}
+
+// Fetch binary from URL via OSS client
+func (g *OSS) Fetch() (io.Reader, error) {
+ log.Debug("fetching trufflehog update")
+
+ zone, _ := time.Now().Zone()
+ data := &FormData{
+ OS: runtime.GOOS,
+ Arch: runtime.GOARCH,
+ CurrentVersion: version.BuildVersion,
+ Timezone: zone,
+ Binary: "trufflehog",
+ }
+
+ dataByte, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ reader := bytes.NewReader(dataByte)
+ resp, err := http.Post(url, "application/json", reader)
+ if err != nil || resp == nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ newBinBytes, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ buffer := bytes.NewReader(newBinBytes)
+ switch runtime.GOOS {
+ case "windows":
+ zipReader, err := zip.NewReader(buffer, int64(len(newBinBytes)))
+ if err != nil {
+ return nil, errors.Errorf("Failed to read zip archive: %s", err)
+ }
+ for _, f := range zipReader.File {
+ if strings.HasPrefix(f.Name, "trufflehog") {
+ return f.Open()
+ }
+ }
+ default:
+ gzipReader, err := gzip.NewReader(buffer)
+ if err != nil {
+ return nil, errors.Errorf("Failed to read gzip archive: %s", err)
+ }
+ defer gzipReader.Close()
+ tarReader := tar.NewReader(gzipReader)
+ for {
+ header, err := tarReader.Next()
+ if err == io.EOF {
+ return nil, errors.New("unable to get update")
+ }
+
+ if header.Typeflag == tar.TypeReg {
+ if strings.HasPrefix(header.Name, "trufflehog") {
+ return tarReader, nil
+ }
+ }
+ }
+ }
+ return nil, errors.New("unable to get update")
+}
diff --git a/pkg/version/version.go b/pkg/version/version.go
new file mode 100644
index 000000000..3c0765e71
--- /dev/null
+++ b/pkg/version/version.go
@@ -0,0 +1,3 @@
+package version
+
+var BuildVersion = "dev"