Skip to content

Find, verify, and analyze leaked credentials

Include line numbers in git metadata (#97)

Bill Richc2b4f0bc39aa
diff --git a/hack/snifftest/main.go b/hack/snifftest/main.go
index 34e8e1fbd..2de774af3 100644
--- a/hack/snifftest/main.go
+++ b/hack/snifftest/main.go
@@ -170,7 +170,7 @@ func main() {
log.Infof("cloned %s", r)
s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
diff --git a/main.go b/main.go
index 481a5a6a0..f7d563dfb 100644
--- a/main.go
+++ b/main.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
+ "gopkg.in/alecthomas/kingpin.v2"
"log"
"net/http"
_ "net/http/pprof"
@@ -17,8 +18,6 @@ import (
"github.com/gorilla/mux"
"github.com/sirupsen/logrus"
- "gopkg.in/alecthomas/kingpin.v2"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 7ad06f0f1..fa5d22420 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -1,7 +1,10 @@
package engine
import (
+ "bytes"
"context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"runtime"
"strings"
"sync"
@@ -174,7 +177,13 @@ func (e *Engine) detectorWorker(ctx context.Context) {
continue
}
for _, result := range results {
- e.results <- detectors.CopyMetadata(chunk, result)
+ targetChunk := chunk
+ if isGitSource(chunk.SourceType) {
+ copyChunk := *chunk
+ targetChunk = &copyChunk
+ SetLineNumber(&copyChunk, &result)
+ }
+ e.results <- detectors.CopyMetadata(targetChunk, result)
}
if len(results) > 0 {
elasped := time.Since(start)
@@ -196,3 +205,51 @@ func (e *Engine) detectorWorker(ctx context.Context) {
atomic.AddUint64(&e.chunksScanned, 1)
}
}
+
+// gitSources is a list of sources that utilize the Git source. It is stored this way because slice consts are not
+// supported.
+func gitSources() []sourcespb.SourceType {
+ return []sourcespb.SourceType{
+ sourcespb.SourceType_SOURCE_TYPE_GIT,
+ sourcespb.SourceType_SOURCE_TYPE_GITHUB,
+ sourcespb.SourceType_SOURCE_TYPE_GITLAB,
+ sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
+ sourcespb.SourceType_SOURCE_TYPE_GERRIT,
+ sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
+ sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
+ }
+}
+
+func isGitSource(sourceType sourcespb.SourceType) bool {
+ for _, i := range gitSources() {
+ if i == sourceType {
+ return true
+ }
+ }
+ return false
+}
+
+func SetLineNumber(chunk *sources.Chunk, result *detectors.Result) {
+ var startingLine *int64
+ switch metadata := chunk.SourceMetadata.GetData().(type) {
+ case *source_metadatapb.MetaData_Git:
+ startingLine = &metadata.Git.Line
+ case *source_metadatapb.MetaData_Github:
+ startingLine = &metadata.Github.Line
+ case *source_metadatapb.MetaData_Gitlab:
+ startingLine = &metadata.Gitlab.Line
+ case *source_metadatapb.MetaData_Bitbucket:
+ startingLine = &metadata.Bitbucket.Line
+ case *source_metadatapb.MetaData_Gerrit:
+ startingLine = &metadata.Gerrit.Line
+ }
+ lines := bytes.Split(chunk.Data, []byte("\n"))
+
+ for i, line := range lines {
+ if bytes.Contains(line, result.Raw) {
+ *startingLine = *startingLine + int64(i)
+ return
+ }
+ }
+ return
+}
diff --git a/pkg/engine/git.go b/pkg/engine/git.go
index 5377b7460..bc2f869ad 100644
--- a/pkg/engine/git.go
+++ b/pkg/engine/git.go
@@ -86,8 +86,8 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
}
scanOptions := git.NewScanOptions(opts...)
- gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "local", true, runtime.NumCPU(),
- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
+ gitSource := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "trufflehog - git", true, runtime.NumCPU(),
+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
@@ -96,6 +96,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
Email: email,
Repository: repository,
Timestamp: timestamp,
+ Line: line,
},
},
}
diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go
index a0feb5be7..834101b15 100644
--- a/pkg/engine/git_test.go
+++ b/pkg/engine/git_test.go
@@ -11,6 +11,11 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
)
+type expResult struct {
+ B string
+ LineNumber int64
+}
+
func TestGitEngine(t *testing.T) {
repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
path, _, err := git.PrepareRepo(repoUrl)
@@ -22,7 +27,7 @@ func TestGitEngine(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
type testProfile struct {
- expected map[string]string
+ expected map[string]expResult
branch string
base string
maxDepth int
@@ -30,17 +35,17 @@ func TestGitEngine(t *testing.T) {
}
for tName, tTest := range map[string]testProfile{
"all_secrets": {
- expected: map[string]string{
- "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",
- "90c75f884c65dc3638ca1610bd9844e668f213c2": "AKIAILE3JG6KMS3HZGCA",
- "8afb0ecd4998b1179e428db5ebbcdc8221214432": "369963c1434c377428ca8531fbc46c0c43d037a0",
- "27fbead3bf883cdb7de9d7825ed401f28f9398f1": "ffc7e0f9400fb6300167009e42d2f842cd7956e2",
+ expected: map[string]expResult{
+ "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
+ "90c75f884c65dc3638ca1610bd9844e668f213c2": {"AKIAILE3JG6KMS3HZGCA", 4},
+ "8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
},
filter: common.FilterEmpty(),
},
"base_commit": {
- expected: map[string]string{
- "70001020fab32b1fcf2f1f0e5c66424eae649826": "AKIAXYZDQCEN4B6JSJQI",
+ expected: map[string]expResult{
+ "70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
},
filter: common.FilterEmpty(),
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -56,8 +61,11 @@ func TestGitEngine(t *testing.T) {
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
- if tTest.expected[meta.Git.Commit] != string(result.Raw) {
- t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit])
+ if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
+ t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
+ }
+ if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
+ t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
}
}
resultCount++
diff --git a/pkg/output/plain.go b/pkg/output/plain.go
index 814d131ec..ea7887dc0 100644
--- a/pkg/output/plain.go
+++ b/pkg/output/plain.go
@@ -42,7 +42,7 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) {
printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))
for _, data := range meta {
for k, v := range data {
- printer.Printf("%s: %s\n", strings.Title(k), v)
+ printer.Printf("%s: %v\n", strings.Title(k), v)
}
}
fmt.Println("")
diff --git a/pkg/pb/source_metadatapb/source_metadata.pb.go b/pkg/pb/source_metadatapb/source_metadata.pb.go
index 95557dd79..7c3d3d9fa 100644
--- a/pkg/pb/source_metadatapb/source_metadata.pb.go
+++ b/pkg/pb/source_metadatapb/source_metadata.pb.go
@@ -113,7 +113,7 @@ type Bitbucket struct {
Email string `protobuf:"bytes,7,opt,name=email,proto3" json:"email,omitempty"`
Link string `protobuf:"bytes,8,opt,name=link,proto3" json:"link,omitempty"`
Timestamp string `protobuf:"bytes,9,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
- Line string `protobuf:"bytes,10,opt,name=line,proto3" json:"line,omitempty"`
+ Line int64 `protobuf:"varint,10,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Bitbucket) Reset() {
@@ -211,11 +211,11 @@ func (x *Bitbucket) GetTimestamp() string {
return ""
}
-func (x *Bitbucket) GetLine() string {
+func (x *Bitbucket) GetLine() int64 {
if x != nil {
return x.Line
}
- return ""
+ return 0
}
type Buildkite struct {
@@ -742,7 +742,7 @@ type Git struct {
Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`
Repository string `protobuf:"bytes,4,opt,name=repository,proto3" json:"repository,omitempty"`
Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
- Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"`
+ Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Git) Reset() {
@@ -812,11 +812,11 @@ func (x *Git) GetTimestamp() string {
return ""
}
-func (x *Git) GetLine() string {
+func (x *Git) GetLine() int64 {
if x != nil {
return x.Line
}
- return ""
+ return 0
}
type Github struct {
@@ -831,7 +831,7 @@ type Github struct {
Email string `protobuf:"bytes,5,opt,name=email,proto3" json:"email,omitempty"`
File string `protobuf:"bytes,6,opt,name=file,proto3" json:"file,omitempty"`
Timestamp string `protobuf:"bytes,7,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
- Line string `protobuf:"bytes,8,opt,name=line,proto3" json:"line,omitempty"`
+ Line int64 `protobuf:"varint,8,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Github) Reset() {
@@ -915,11 +915,11 @@ func (x *Github) GetTimestamp() string {
return ""
}
-func (x *Github) GetLine() string {
+func (x *Github) GetLine() int64 {
if x != nil {
return x.Line
}
- return ""
+ return 0
}
type Gitlab struct {
@@ -933,7 +933,7 @@ type Gitlab struct {
Email string `protobuf:"bytes,4,opt,name=email,proto3" json:"email,omitempty"`
Repository string `protobuf:"bytes,5,opt,name=repository,proto3" json:"repository,omitempty"`
Timestamp string `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
- Line string `protobuf:"bytes,7,opt,name=line,proto3" json:"line,omitempty"`
+ Line int64 `protobuf:"varint,7,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Gitlab) Reset() {
@@ -1010,11 +1010,11 @@ func (x *Gitlab) GetTimestamp() string {
return ""
}
-func (x *Gitlab) GetLine() string {
+func (x *Gitlab) GetLine() int64 {
if x != nil {
return x.Line
}
- return ""
+ return 0
}
type GCS struct {
@@ -1517,7 +1517,7 @@ type Gerrit struct {
Email string `protobuf:"bytes,3,opt,name=email,proto3" json:"email,omitempty"`
Project string `protobuf:"bytes,4,opt,name=project,proto3" json:"project,omitempty"` // projects are what Gerrit calls repositories
Timestamp string `protobuf:"bytes,5,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
- Line string `protobuf:"bytes,6,opt,name=line,proto3" json:"line,omitempty"`
+ Line int64 `protobuf:"varint,6,opt,name=line,proto3" json:"line,omitempty"`
}
func (x *Gerrit) Reset() {
@@ -1587,11 +1587,11 @@ func (x *Gerrit) GetTimestamp() string {
return ""
}
-func (x *Gerrit) GetLine() string {
+func (x *Gerrit) GetLine() int64 {
if x != nil {
return x.Line
}
- return ""
+ return 0
}
type Test struct {
@@ -2073,7 +2073,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b,
0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12,
- 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69,
+ 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69,
0x6e, 0x65, 0x22, 0xa4, 0x01, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65,
0x12, 0x10, 0x0a, 0x03, 0x6f, 0x72, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6f,
0x72, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02,
@@ -2140,7 +2140,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65,
- 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,
+ 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xcc, 0x01, 0x0a,
0x06, 0x47, 0x69, 0x74, 0x68, 0x75, 0x62, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x75,
0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75,
@@ -2153,7 +2153,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d,
0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x69,
0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18,
- 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,
+ 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x06,
0x47, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x12, 0x12,
0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69,
@@ -2164,7 +2164,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x52, 0x0a, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09,
0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69,
- 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,
+ 0x6e, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x5b,
0x0a, 0x03, 0x47, 0x43, 0x53, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a,
0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c,
@@ -2224,7 +2224,7 @@ var file_source_metadata_proto_rawDesc = []byte{
0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74,
0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e,
- 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,
+ 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x22, 0x1a, 0x0a,
0x04, 0x54, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x81, 0x01, 0x0a, 0x07, 0x4a, 0x65,
0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74,
diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go
index ecc1e9ce1..1911c7f41 100644
--- a/pkg/sources/git/git.go
+++ b/pkg/sources/git/git.go
@@ -47,14 +47,14 @@ type Git struct {
sourceName string
sourceID int64
jobID int64
- sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData
+ sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
verify bool
// sem is used to limit concurrency
sem *semaphore.Weighted
}
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
- sourceMetadataFunc func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData,
+ sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
) *Git {
return &Git{
sourceType: sourceType,
@@ -102,7 +102,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn
s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, runtime.NumCPU(),
- func(file, email, commit, repository, timestamp string) *source_metadatapb.MetaData {
+ func(file, email, commit, repository, timestamp string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Git{
Git: &source_metadatapb.Git{
@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Email: sanitizer.UTF8(email),
Repository: sanitizer.UTF8(repository),
Timestamp: sanitizer.UTF8(timestamp),
+ Line: line,
},
},
}
@@ -269,14 +270,7 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
if !scanOptions.Filter.Pass(file.NewName) {
continue
}
- newLines := bytes.Buffer{}
- for _, frag := range file.TextFragments {
- for _, line := range frag.Lines {
- if line.Op == gitdiff.OpAdd {
- newLines.WriteString(line.String())
- }
- }
- }
+
fileName := file.NewName
if fileName == "" {
continue
@@ -299,14 +293,23 @@ func (s *Git) ScanCommits(repo *git.Repository, path string, scanOptions *ScanOp
return errors.WrapPrefix(err, "couldn't get repo name", 0)
}
- metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo)
- chunksChan <- &sources.Chunk{
- SourceName: s.sourceName,
- SourceID: s.sourceID,
- SourceType: s.sourceType,
- SourceMetadata: metadata,
- Data: newLines.Bytes(),
- Verify: s.verify,
+ for _, frag := range file.TextFragments {
+ newLines := bytes.Buffer{}
+ newLineNumber := frag.NewPosition
+ for _, line := range frag.Lines {
+ if line.Op == gitdiff.OpAdd {
+ newLines.WriteString(strings.ReplaceAll(line.String(), "\n", " ") + "\n")
+ }
+ }
+ metadata := s.sourceMetadataFunc(fileName, email, hash, when, safeRepo, newLineNumber)
+ chunksChan <- &sources.Chunk{
+ SourceName: s.sourceName,
+ SourceID: s.sourceID,
+ SourceType: s.sourceType,
+ SourceMetadata: metadata,
+ Data: newLines.Bytes(),
+ Verify: s.verify,
+ }
}
}
return nil
@@ -341,7 +344,7 @@ func (s *Git) ScanUnstaged(repo *git.Repository, scanOptions *ScanOptions, chunk
continue
}
metadata := s.sourceMetadataFunc(
- fh, "unstaged", "unstaged", time.Now().String(), safeRepo,
+ fh, "unstaged", "unstaged", time.Now().String(), safeRepo, 0,
)
fileBuf := bytes.NewBuffer(nil)
diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.go
index 06de20231..61b9da20d 100644
--- a/pkg/sources/git/git_test.go
+++ b/pkg/sources/git/git_test.go
@@ -149,6 +149,7 @@ func TestSource_Chunks_Integration(t *testing.T) {
type byteCompare struct {
B []byte
Found bool
+ Multi bool
}
tests := []struct {
name string
@@ -168,17 +169,17 @@ func TestSource_Chunks_Integration(t *testing.T) {
},
},
expectedChunkData: map[string]*byteCompare{
- "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default]\n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI\n+aws_secret_access_key = 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\n+output = json\n+region = us-east-2\n")},
- "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f\n")},
- "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},
- "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},
- "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},
- "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops\n")},
- "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+\n")},
- "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that\n+\n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\"\n+\n+oh, goodness. there's another one!")},
- "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here\n+\n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\"\n+\n+yup, just did that")},
- "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj\n+\n+this is the secret: AKIA2E0A8F3B244C9986\n+\n+okay thank you bye")},
- "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default]\n+Access key Id: AKIAILE3JG6KMS3HZGCA\n+Secret Access Key: 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\n+okay thank you bye\n")},
+ "70001020fab32b1fcf2f1f0e5c66424eae649826-aws": {B: []byte("+[default] \n+aws_access_key_id = AKIAXYZDQCEN4B6JSJQI \n+aws_secret_access_key = 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 \n+output = json \n+region = us-east-2 \n")},
+ "a6f8aa55736d4a85be31a0048a4607396898647a-bump": {B: []byte("+f \n")},
+ "07d96d011005fe8296bdd237c13a06a72e96783d-bump": {B: []byte("+ s \n")},
+ "2f251b8c1e72135a375b659951097ec7749d4af9-bump": {B: []byte("+ \n")},
+ "e6c8bbabd8796ea3cd85bfc2e55b27e0a491747f-bump": {B: []byte("+oops \n")},
+ "735b52b0eb40610002bb1088e902bd61824eb305-bump": {B: []byte("+oops \n")},
+ "ce62d79908803153ef6e145e042d3e80488ef747-bump": {B: []byte("+ \n")},
+ "27fbead3bf883cdb7de9d7825ed401f28f9398f1-slack": {B: []byte("+yup, just did that \n+ \n+github_lol: \"ffc7e0f9400fb6300167009e42d2f842cd7956e2\" \n+ \n+oh, goodness. there's another one!\n")},
+ "8afb0ecd4998b1179e428db5ebbcdc8221214432-slack": {B: []byte("+oops might drop a slack token here \n+ \n+github_secret=\"369963c1434c377428ca8531fbc46c0c43d037a0\" \n+ \n+yup, just did that\n"), Multi: true},
+ "8fe6f04ef1839e3fc54b5147e3d0e0b7ab971bd5-aws": {B: []byte("+blah blaj \n+ \n+this is the secret: AKIA2E0A8F3B244C9986 \n+ \n+okay thank you bye\n"), Multi: true},
+ "90c75f884c65dc3638ca1610bd9844e668f213c2-aws": {B: []byte("+this is the secret: [Default] \n+Access key Id: AKIAILE3JG6KMS3HZGCA \n+Secret Access Key: 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 \n"), Multi: true},
},
},
}
@@ -216,8 +217,9 @@ func TestSource_Chunks_Integration(t *testing.T) {
if expectedData, exists := tt.expectedChunkData[key]; !exists {
t.Errorf("A chunk exists that was not expected with key %q", key)
} else {
- (*tt.expectedChunkData[key]).Found = true
- if !bytes.Equal(chunk.Data, expectedData.B) {
+ if bytes.Equal(chunk.Data, expectedData.B) {
+ (*tt.expectedChunkData[key]).Found = true
+ } else if !expectedData.Multi {
t.Errorf("Got %q: %q, which was not expected", key, string(chunk.Data))
}
}
diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go
index 5ad28853f..231171c3d 100644
--- a/pkg/sources/github/github.go
+++ b/pkg/sources/github/github.go
@@ -113,7 +113,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
s.conn = &conn
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Github{
Github: &source_metadatapb.Github{
@@ -123,6 +123,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Repository: sanitizer.UTF8(repository),
Link: git.GenerateLink(repository, commit, file),
Timestamp: sanitizer.UTF8(timestamp),
+ Line: line,
},
},
}
diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.go
index 330254206..f3643621e 100644
--- a/pkg/sources/gitlab/gitlab.go
+++ b/pkg/sources/gitlab/gitlab.go
@@ -101,7 +101,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
}
s.git = git.NewGit(s.Type(), s.JobID(), s.SourceID(), s.name, s.verify, runtime.NumCPU(),
- func(file, email, commit, timestamp, repository string) *source_metadatapb.MetaData {
+ func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
return &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_Gitlab{
Gitlab: &source_metadatapb.Gitlab{
@@ -111,6 +111,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
Repository: sanitizer.UTF8(repository),
Link: git.GenerateLink(repository, commit, file),
Timestamp: sanitizer.UTF8(timestamp),
+ Line: line,
},
},
}
diff --git a/proto/source_metadata.proto b/proto/source_metadata.proto
index 0f9b5518e..39b5d3a74 100644
--- a/proto/source_metadata.proto
+++ b/proto/source_metadata.proto
@@ -22,7 +22,7 @@ message Bitbucket {
string email = 7;
string link = 8;
string timestamp = 9;
- string line = 10;
+ int64 line = 10;
}
message Buildkite {
@@ -84,7 +84,7 @@ message Git {
string email = 3;
string repository = 4;
string timestamp = 5;
- string line = 6;
+ int64 line = 6;
}
message Github {
@@ -95,7 +95,7 @@ message Github {
string email = 5;
string file = 6;
string timestamp = 7;
- string line = 8;
+ int64 line = 8;
}
message Gitlab {
@@ -105,7 +105,7 @@ message Gitlab {
string email = 4;
string repository = 5;
string timestamp = 6;
- string line = 7;
+ int64 line = 7;
}
message GCS {
@@ -164,7 +164,7 @@ message Gerrit {
string email = 3;
string project = 4; // projects are what Gerrit calls repositories
string timestamp = 5;
- string line = 6;
+ int64 line = 6;
}
message Test {