Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Include line numbers in git metadata (#97)
Bill Richc2b4f0bc39aa
diff --git a/hack/snifftest/main.go b/hack/snifftest/main.goindex 34e8e1fbd..2de774af3 100644--- a/hack/snifftest/main.go+++ b/hack/snifftest/main.go@@ -170,7 +170,7 @@ func main() {log.Infof("cloned %s", r)s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Git{Git: &source_metadatapb.Git{diff --git a/main.go b/main.goindex 481a5a6a0..f7d563dfb 100644--- a/main.go+++ b/main.go@@ -4,6 +4,7 @@ import ("context""encoding/json""fmt"+ "gopkg.in/alecthomas/kingpin.v2""log""net/http"_ "net/http/pprof"@@ -17,8 +18,6 @@ import ("github.com/gorilla/mux""github.com/sirupsen/logrus"- "gopkg.in/alecthomas/kingpin.v2"-"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/engine"diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 7ad06f0f1..fa5d22420 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -1,7 +1,10 @@package engineimport (+ "bytes""context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb""runtime""strings""sync"@@ -174,7 +177,13 @@ func (e *Engine) detectorWorker(ctx context.Context) {continue}for _, result := range results {- e.results <- detectors.CopyMetadata(chunk, result)+ targetChunk := chunk+ if isGitSource(chunk.SourceType) {+ copyChunk := *chunk+ targetChunk = ©Chunk+ SetLineNumber(©Chunk, &result)+ }+ e.results <- detectors.CopyMetadata(targetChunk, result)}if len(results) > 0 {elasped := time.Since(start)@@ -196,3 +205,51 @@ func (e *Engine) detectorWorker(ctx context.Context) {atomic.AddUint64(&e.chunksScanned, 1)}}++// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not+// supported.+func gitSources() []sourcespb.SourceType {+ return []sourcespb.SourceType{+ sourcespb.SourceType_SOURCE_TYPE_GIT,+ sourcespb.SourceType_SOURCE_TYPE_GITHUB,+ sourcespb.SourceType_SOURCE_TYPE_GITLAB,+ sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,+ sourcespb.SourceType_SOURCE_TYPE_GERRIT,+ sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,+ sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,+ }+}++func isGitSource(sourceType sourcespb.SourceType) bool {+ for _, i := range gitSources() {+ if i == sourceType {+ return true+ }+ }+ return false+}++func SetLineNumber(chunk *sources.Chunk, result *detectors.Result) {+ var startingLine *int64+ switch metadata := chunk.SourceMetadata.GetData().(type) {+ case *source_metadatapb.MetaData_Git:+ startingLine = &metadata.Git.Line+ case *source_metadatapb.MetaData_Github:+ startingLine = &metadata.Github.Line+ case *source_metadatapb.MetaData_Gitlab:+ startingLine = &metadata.Gitlab.Line+ case *source_metadatapb.MetaData_Bitbucket:+ startingLine = &metadata.Bitbucket.Line+ case *source_metadatapb.MetaData_Gerrit:+ startingLine = &metadata.Gerrit.Line+ }+ lines := bytes.Split(chunk.Data, []byte("\n"))++ for i, line := range lines {+ if bytes.Contains(line, result.Raw) {+ *startingLine = *startingLine + int64(i)+ return+ }+ }+ return+}diff --git a/pkg/engine/git.go b/pkg/engine/git.goindex 5377b7460..bc2f869ad 100644--- a/pkg/engine/git.go+++ b/pkg/engine/git.go@@ -86,8 +86,8 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,}scanOptions := git.NewScanOptions(opts...)- gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(),- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {+ gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Git{Git: &source_metadatapb.Git{@@ -96,6 +96,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,Email: email,Repository: repository,Timestamp: timestamp,+ Line: line,},},}diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.goindex a0feb5be7..834101b15 100644--- a/pkg/engine/git_test.go+++ b/pkg/engine/git_test.go@@ -11,6 +11,11 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources/git")+type expResult struct {+ B string+ LineNumber int64+}+func TestGitEngine(t *testing.T) {repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"path, _, err := git.PrepareRepo(repoUrl)@@ -22,7 +27,7 @@ func TestGitEngine(t *testing.T) {ctx, cancel := context.WithCancel(context.TODO())defer cancel()type testProfile struct {- expected map[string]string+ expected map[string]expResultbranch stringbase stringmaxDepth int@@ -30,17 +35,17 @@ func TestGitEngine(t *testing.T) {}for tName, tTest := range map[string]testProfile{"all_secrets": {- expected: map[string]string{- "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",- "90c75f884c65dc3638ca1610bd9844e668f213c2": "AKIAILE3JG6KMS3HZGCA",- "8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0",- "27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2",+ expected: map[string]expResult{+ "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},+ "90c75f884c65dc3638ca1610bd9844e668f213c2": {"AKIAILE3JG6KMS3HZGCA", 4},+ "8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},},filter: common.FilterEmpty(),},"base_commit": {- expected: map[string]string{- "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",+ expected: map[string]expResult{+ "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},},filter: common.FilterEmpty(),base: "2f251b8c1e72135a375b659951097ec7749d4af9",@@ -56,8 +61,11 @@ func TestGitEngine(t *testing.T) {for result := range e.ResultsChan() {switch meta := result.SourceMetadata.GetData().(type) {case *source_metadatapb.MetaData_Git:- if tTest.expected[meta.Git.Commit] != string(result.Raw) {- t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit])+ if tTest.expected[meta.Git.Commit].B != string(result.Raw) {+ t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)+ }+ if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {+ t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)}}resultCount++diff --git a/pkg/output/plain.go b/pkg/output/plain.goindex 814d131ec..ea7887dc0 100644--- a/pkg/output/plain.go+++ b/pkg/output/plain.go@@ -42,7 +42,7 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) {printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))for _, data := range meta {for k, v := range data {- printer.Printf("%s: %s\n", strings.Title(k), v)+ printer.Printf("%s: %v\n", strings.Title(k), v)}}fmt.Println("")diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.go b/pkg/pb/source_metadatapb/source_metadata.pb.goindex 95557dd79..7c3d3d9fa 100644--- a/pkg/pb/source_metadatapb/source_metadata.pb.go+++ b/pkg/pb/source_metadatapb/source_metadata.pb.go@@ -113,7 +113,7 @@ type Bitbucket struct {Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"`Link string `protobuf:"bytes,8,opt,name=link,proto3" json:"link,omitempty"`Timestamp string `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"`- Line string `protobuf:"bytes,10,opt,name=line,proto3" json:"line,omitempty"`+ Line int64 `protobuf:"varint,10,opt,name=line,proto3" json:"line,omitempty"`}func (x *Bitbucket) Reset() {@@ -211,11 +211,11 @@ func (x *Bitbucket) GetTimestamp() string {return ""}-func (x *Bitbucket) GetLine() string {+func (x *Bitbucket) GetLine() int64 {if x != nil {return x.Line}- return ""+ return 0}type Buildkite struct {@@ -742,7 +742,7 @@ type Git struct {Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`Repository string `protobuf:"bytes,4,opt,name=repository,proto3" json:"repository,omitempty"`Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`- Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"`+ Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`}func (x *Git) Reset() {@@ -812,11 +812,11 @@ func (x *Git) GetTimestamp() string {return ""}-func (x *Git) GetLine() string {+func (x *Git) GetLine() int64 {if x != nil {return x.Line}- return ""+ return 0}type Github struct {@@ -831,7 +831,7 @@ type Github struct {Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`- Line string `protobuf:"bytes,8,opt,name=line,proto3" json:"line,omitempty"`+ Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`}func (x *Github) Reset() {@@ -915,11 +915,11 @@ func (x *Github) GetTimestamp() string {return ""}-func (x *Github) GetLine() string {+func (x *Github) GetLine() int64 {if x != nil {return x.Line}- return ""+ return 0}type Gitlab struct {@@ -933,7 +933,7 @@ type Gitlab struct {Email string `protobuf:"bytes,4,opt,name=email,proto3" json:"email,omitempty"`Repository string `protobuf:"bytes,5,opt,name=repository,proto3" json:"repository,omitempty"`Timestamp string `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"`- Line string `protobuf:"bytes,7,opt,name=line,proto3" json:"line,omitempty"`+ Line int64 `protobuf:"varint,7,opt,name=line,proto3" json:"line,omitempty"`}func (x *Gitlab) Reset() {@@ -1010,11 +1010,11 @@ func (x *Gitlab) GetTimestamp() string {return ""}-func (x *Gitlab) GetLine() string {+func (x *Gitlab) GetLine() int64 {if x != nil {return x.Line}- return ""+ return 0}type GCS struct {@@ -1517,7 +1517,7 @@ type Gerrit struct {Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`Project string `protobuf:"bytes,4,opt,name=project,proto3" json:"project,omitempty"` // projects are what Gerrit calls repositoriesTimestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`- Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"`+ Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`}func (x *Gerrit) Reset() {@@ -1587,11 +1587,11 @@ func (x *Gerrit) GetTimestamp() string {return ""}-func (x *Gerrit) GetLine() string {+func (x *Gerrit) GetLine() int64 {if x != nil {return x.Line}- return ""+ return 0}type Test struct {@@ -2073,7 +2073,7 @@ var file_source_metadata_proto_rawDesc = []byte{0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b,0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20,0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12,- 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,+ 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69,0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65,0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f,0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02,@@ -2140,7 +2140,7 @@ var file_source_metadata_proto_rawDesc = []byte{0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69,0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74,0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65,- 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,+ 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18,0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75,0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,@@ -2153,7 +2153,7 @@ var file_source_metadata_proto_rawDesc = []byte{0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d,0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69,0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18,- 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,+ 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12,0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69,@@ -2164,7 +2164,7 @@ var file_source_metadata_proto_rawDesc = []byte{0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09,0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52,0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69,- 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,+ 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18,0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a,0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c,@@ -2224,7 +2224,7 @@ var file_source_metadata_proto_rawDesc = []byte{0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74,0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e,- 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,+ 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20,0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65,0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex ecc1e9ce1..1911c7f41 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -47,14 +47,14 @@ type Git struct {sourceName stringsourceID int64jobID int64- sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData+ sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaDataverify bool// sem is used to limit concurrencysem *semaphore.Weighted}func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,- sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData,+ sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,) *Git {return &Git{sourceType: sourceType,@@ -102,7 +102,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,s.conn = &conns.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(),- func(file, email, commit, repository, timestamp string) *source_metadatapb.MetaData {+ func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Git{Git: &source_metadatapb.Git{@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,Email: sanitizer.UTF8(email),Repository: sanitizer.UTF8(repository),Timestamp: sanitizer.UTF8(timestamp),+ Line: line,},},}@@ -269,14 +270,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOpif !scanOptions.Filter.Pass(file.NewName) {continue}- newLines := bytes.Buffer{}- for _, frag := range file.TextFragments {- for _, line := range frag.Lines {- if line.Op == gitdiff.OpAdd {- newLines.WriteString(line.String())- }- }- }+fileName := file.NewNameif fileName == "" {continue@@ -299,14 +293,23 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOpreturn errors.WrapPrefix(err, "couldn't get repo name", 0)}- metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo)- chunksChan <- &sources.Chunk{- SourceName: s.sourceName,- SourceID: s.sourceID,- SourceType: s.sourceType,- SourceMetadata: metadata,- Data: newLines.Bytes(),- Verify: s.verify,+ for _, frag := range file.TextFragments {+ newLines := bytes.Buffer{}+ newLineNumber := frag.NewPosition+ for _, line := range frag.Lines {+ if line.Op == gitdiff.OpAdd {+ newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n")+ }+ }+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo, newLineNumber)+ chunksChan <- &sources.Chunk{+ SourceName: s.sourceName,+ SourceID: s.sourceID,+ SourceType: s.sourceType,+ SourceMetadata: metadata,+ Data: newLines.Bytes(),+ Verify: s.verify,+ }}}return nil@@ -341,7 +344,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunkcontinue}metadata := s.sourceMetadataFunc(- fh, "unstaged", "unstaged", time.Now().String(), safeRepo,+ fh, "unstaged", "unstaged", time.Now().String(), safeRepo, 0,)fileBuf := bytes.NewBuffer(nil)diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 06de20231..61b9da20d 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -149,6 +149,7 @@ func TestSource_Chunks_Integration(t *testing.T) {type byteCompare struct {B []byteFound bool+ Multi bool}tests := []struct {name string@@ -168,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) {},},expectedChunkData: map[string]*byteCompare{- "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default]\n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI\n+aws_secret_access_key = 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\n+output = json\n+region = us-east-2\n")},- "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f\n")},- "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},- "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},- "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},- "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops\n")},- "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+\n")},- "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that\n+\n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n+\n+oh, goodness. there's another one!")},- "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here\n+\n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n+\n+yup, just did that")},- "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj\n+\n+this is the secret: AKIA2E0A8F3B244C9986\n+\n+okay thank you bye")},- "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default]\n+Access key Id: AKIAILE3JG6KMS3HZGCA\n+Secret Access Key: 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\n+okay thank you bye\n")},+ "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")},+ "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f \n")},+ "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},+ "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},+ "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},+ "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops \n")},+ "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+ \n")},+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")},+ "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true},+ "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true},+ "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true},},},}@@ -216,8 +217,9 @@ func TestSource_Chunks_Integration(t *testing.T) {if expectedData, exists := tt.expectedChunkData[key]; !exists {t.Errorf("A chunk exists that was not expected with key %q", key)} else {- (*tt.expectedChunkData[key]).Found = true- if !bytes.Equal(chunk.Data, expectedData.B) {+ if bytes.Equal(chunk.Data, expectedData.B) {+ (*tt.expectedChunkData[key]).Found = true+ } else if !expectedData.Multi {t.Errorf("Got %q: %q, which was not expected", key, string(chunk.Data))}}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 5ad28853f..231171c3d 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -113,7 +113,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,s.conn = &conns.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Github{Github: &source_metadatapb.Github{@@ -123,6 +123,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,Repository: sanitizer.UTF8(repository),Link: git.GenerateLink(repository, commit, file),Timestamp: sanitizer.UTF8(timestamp),+ Line: line,},},}diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 330254206..f3643621e 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -101,7 +101,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,}s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {return &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Gitlab{Gitlab: &source_metadatapb.Gitlab{@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,Repository: sanitizer.UTF8(repository),Link: git.GenerateLink(repository, commit, file),Timestamp: sanitizer.UTF8(timestamp),+ Line: line,},},}diff --git a/proto/source_metadata.proto b/proto/source_metadata.protoindex 0f9b5518e..39b5d3a74 100644--- a/proto/source_metadata.proto+++ b/proto/source_metadata.proto@@ -22,7 +22,7 @@ message Bitbucket {string email = 7;string link = 8;string timestamp = 9;- string line = 10;+ int64 line = 10;}message Buildkite {@@ -84,7 +84,7 @@ message Git {string email = 3;string repository = 4;string timestamp = 5;- string line = 6;+ int64 line = 6;}message Github {@@ -95,7 +95,7 @@ message Github {string email = 5;string file = 6;string timestamp = 7;- string line = 8;+ int64 line = 8;}message Gitlab {@@ -105,7 +105,7 @@ message Gitlab {string email = 4;string repository = 5;string timestamp = 6;- string line = 7;+ int64 line = 7;}message GCS {@@ -164,7 +164,7 @@ message Gerrit {string email = 3;string project = 4; // projects are what Gerrit calls repositoriesstring timestamp = 5;- string line = 6;+ int64 line = 6;}message Test {