Skip to content

Find, verify, and analyze leaked credentials

Ability to update line number in link (#1788)

ahrav47d5ddebf2b9
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index ff6167905..04979d3ac 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -19,6 +19,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -532,12 +533,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
}
for _, res := range results {
- e.processResult(data, res)
+ e.processResult(ctx, data, res)
}
data.wgDoneFn()
}
-func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
+func (e *Engine) processResult(ctx context.Context, data detectableChunk, res detectors.Result) {
ignoreLinePresent := false
if SupportsLineNumbers(data.chunk.SourceType) {
copyChunk := data.chunk
@@ -545,8 +546,11 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
copyChunk.SourceMetadata = copyMetaData
}
- fragStart, mdLine := FragmentFirstLine(&copyChunk)
+ fragStart, mdLine, link := FragmentFirstLineAndLink(&copyChunk)
ignoreLinePresent = SetResultLineNumber(&copyChunk, &res, fragStart, mdLine)
+ if link != "" {
+ copyChunk.SourceMetadata.GetGithub().Link = giturl.UpdateLinkLineNumber(ctx, link, *mdLine)
+ }
data.chunk = copyChunk
}
if ignoreLinePresent {
@@ -625,27 +629,42 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
return lineNumber, false
}
-// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
-// chunk metadata.
-func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
- var fragmentStart *int64
+// FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata.
+// It returns:
+// - The first line number of the fragment.
+// - A pointer to the line number, facilitating direct updates.
+// - The link associated with the fragment. This link may be updated in the chunk metadata
+// if there's a change in the line number.
+func FragmentFirstLineAndLink(chunk *sources.Chunk) (int64, *int64, string) {
+ if chunk.SourceMetadata == nil {
+ return 0, nil, ""
+ }
+
+ var (
+ fragmentStart *int64
+ link string
+ )
switch metadata := chunk.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
fragmentStart = &metadata.Git.Line
case *source_metadatapb.MetaData_Github:
fragmentStart = &metadata.Github.Line
+ link = metadata.Github.Link
case *source_metadatapb.MetaData_Gitlab:
fragmentStart = &metadata.Gitlab.Line
+ link = metadata.Gitlab.Link
case *source_metadatapb.MetaData_Bitbucket:
fragmentStart = &metadata.Bitbucket.Line
+ link = metadata.Bitbucket.Link
case *source_metadatapb.MetaData_Gerrit:
fragmentStart = &metadata.Gerrit.Line
case *source_metadatapb.MetaData_Filesystem:
fragmentStart = &metadata.Filesystem.Line
+ link = metadata.Filesystem.Link
default:
- return 0, nil
+ return 0, nil, ""
}
- return *fragmentStart, fragmentStart
+ return *fragmentStart, fragmentStart, link
}
// SetResultLineNumber sets the line number in the provided result.
diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go
index 948d7f947..6c3affcf3 100644
--- a/pkg/engine/engine_test.go
+++ b/pkg/engine/engine_test.go
@@ -11,6 +11,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -205,3 +206,60 @@ func TestEngine_DuplicatSecrets(t *testing.T) {
want := uint64(5)
assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
}
+
+func TestFragmentFirstLineAndLink(t *testing.T) {
+ tests := []struct {
+ name string
+ chunk *sources.Chunk
+ expectedLine int64
+ expectedLink string
+ }{
+ {
+ name: "Test Git Metadata",
+ chunk: &sources.Chunk{
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Git{
+ Git: &source_metadatapb.Git{
+ Line: 10,
+ },
+ },
+ },
+ },
+ expectedLine: 10,
+ expectedLink: "", // Git doesn't support links
+ },
+ {
+ name: "Test Github Metadata",
+ chunk: &sources.Chunk{
+ SourceMetadata: &source_metadatapb.MetaData{
+ Data: &source_metadatapb.MetaData_Github{
+ Github: &source_metadatapb.Github{
+ Line: 5,
+ Link: "https://example.github.com",
+ },
+ },
+ },
+ },
+ expectedLine: 5,
+ expectedLink: "https://example.github.com",
+ },
+ {
+ name: "Unsupported Type",
+ chunk: &sources.Chunk{},
+ expectedLine: 0,
+ expectedLink: "",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ line, linePtr, link := FragmentFirstLineAndLink(tt.chunk)
+ assert.Equal(t, tt.expectedLink, link, "Mismatch in link")
+ assert.Equal(t, tt.expectedLine, line, "Mismatch in line")
+
+ if linePtr != nil {
+ assert.Equal(t, tt.expectedLine, *linePtr, "Mismatch in linePtr value")
+ }
+ })
+ }
+}
diff --git a/pkg/giturl/giturl.go b/pkg/giturl/giturl.go
index b9c1bcec2..bf7adee04 100644
--- a/pkg/giturl/giturl.go
+++ b/pkg/giturl/giturl.go
@@ -2,10 +2,13 @@ package giturl
import (
"net/url"
+ "regexp"
"strconv"
"strings"
"github.com/pkg/errors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
type provider string
@@ -143,3 +146,43 @@ func GenerateLink(repo, commit, file string, line int64) string {
return baseLink
}
}
+
+var linePattern = regexp.MustCompile(`L\d+`)
+
+// UpdateLinkLineNumber updates the line number in a repository link.
+// Used post-link generation to refine reported issue locations within large scanned blocks.
+func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) string {
+ parsedURL, err := url.Parse(link)
+ if err != nil {
+ ctx.Logger().Error(err, "unable to parse link to update line number", "link", link)
+ return link
+ }
+
+ switch determineProvider(link) {
+ case providerBitbucket:
+ // For Bitbucket, it doesn't support line links (based on the GenerateLink function).
+ // So we don't need to change anything.
+ return link
+
+ case providerAzure:
+ // For Azure, line numbers are appended as ?line=<number>.
+ query := parsedURL.Query()
+ query.Set("line", strconv.FormatInt(newLine, 10))
+ parsedURL.RawQuery = query.Encode()
+
+ case providerGithub, providerGitlab:
+ // If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
+ // So do the same thing.
+ fallthrough
+ default:
+ // Assumed format: .../blob/<commit>/file.go#L<number>
+ fragment := "L" + strconv.FormatInt(newLine, 10)
+ if linePattern.MatchString(parsedURL.Fragment) {
+ parsedURL.Fragment = linePattern.ReplaceAllString(parsedURL.Fragment, fragment)
+ } else {
+ parsedURL.Fragment += fragment
+ }
+ }
+
+ return parsedURL.String()
+}
diff --git a/pkg/giturl/giturl_test.go b/pkg/giturl/giturl_test.go
index a59c284d0..1ccc2fd74 100644
--- a/pkg/giturl/giturl_test.go
+++ b/pkg/giturl/giturl_test.go
@@ -4,6 +4,8 @@ import (
"testing"
"github.com/pkg/errors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
)
func Test_NormalizeOrgRepoURL(t *testing.T) {
@@ -187,3 +189,84 @@ func TestGenerateLink(t *testing.T) {
})
}
}
+
+func TestUpdateLinkLineNumber(t *testing.T) {
+ t.Parallel()
+
+ type args struct {
+ link string
+ newLine int64
+ }
+ tests := []struct {
+ name string
+ args args
+ want string
+ wantErr bool
+ }{
+ {
+ name: "Update bitbucket, no line number supported",
+ args: args{
+ link: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
+ newLine: int64(10),
+ },
+ want: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
+ },
+ {
+ name: "Update github link with line",
+ args: args{
+ link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L4",
+ newLine: int64(10),
+ },
+ want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L10",
+ },
+ {
+ name: "Update Azure link with line",
+ args: args{
+ link: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
+ newLine: int64(40),
+ },
+ want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=40",
+ },
+ {
+ name: "Add line to github link without line",
+ args: args{
+ link: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore",
+ newLine: int64(7),
+ },
+ want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L7",
+ },
+ {
+ name: "Update Unknown provider on-prem instance with line",
+ args: args{
+ link: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
+ newLine: int64(50),
+ },
+ want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L50",
+ },
+ {
+ name: "Update Unknown provider on-prem instance without line",
+ args: args{
+ link: "https://onprem.customdomain.com/org/repo/commit/xyz123",
+ newLine: int64(50),
+ },
+ want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
+ },
+ {
+ name: "Invalid link",
+ args: args{
+ link: "definitely not a link",
+ newLine: int64(50),
+ },
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := UpdateLinkLineNumber(context.Background(), tt.args.link, tt.args.newLine)
+ if got != tt.want && !tt.wantErr {
+ t.Errorf("UpdateLinkLineNumber() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}