Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fix line numbers for duplicate secrets within a chunk (#4910)
Amaan Ullah0ec3634f6cf6
diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.goindex a8b435e54..760971b68 100644--- a/pkg/custom_detectors/custom_detectors_test.go+++ b/pkg/custom_detectors/custom_detectors_test.go@@ -632,7 +632,10 @@ func TestDetectorValidations(t *testing.T) {results, err := detector.FromData(context.Background(), false, []byte(tt.input.Data))assert.NoError(t, err)- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "ExtraData", "verificationError", "primarySecret")+ ignoreOpts := cmp.Options{+ cmpopts.IgnoreUnexported(detectors.Result{}),+ cmpopts.IgnoreFields(detectors.Result{}, "ExtraData"),+ }if diff := cmp.Diff(results, tt.want, ignoreOpts); diff != "" {t.Errorf("CustomDetector.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/datadogapikey/datadogapikey_test.go b/pkg/detectors/datadogapikey/datadogapikey_test.goindex f5918a4d1..a6495bcad 100644--- a/pkg/detectors/datadogapikey/datadogapikey_test.go+++ b/pkg/detectors/datadogapikey/datadogapikey_test.go@@ -41,7 +41,7 @@ func TestDataDogApiKey_Pattern_WithValidAPIKey(t *testing.T) {return}- if diff := cmp.Diff(wantedResult, results, cmpopts.IgnoreFields(detectors.Result{}, "verificationError", "primarySecret")); diff != "" {+ if diff := cmp.Diff(wantedResult, results, cmpopts.IgnoreUnexported(detectors.Result{})); diff != "" {t.Errorf("%s diff: (-want +got)\n%s", "TestDataDogApiKey_Pattern_WithValidAPIKeyOnly", diff)}}diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex fc781af1d..9efb88d64 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -134,6 +134,11 @@ type Result struct {Value stringLine int64}++ // chunkOffset stores the byte position of this result's secret within chunk data.+ // Used to disambiguate line numbers when the same secret appears multiple times.+ chunkOffset int64+ chunkOffsetSet bool}// CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when@@ -176,6 +181,22 @@ func (r *Result) GetPrimarySecretValue() string {return r.primarySecret.Value}+// SetChunkOffset records the byte position of this result's secret within the chunk data.+func (r *Result) SetChunkOffset(offset int64) {+ r.chunkOffset = offset+ r.chunkOffsetSet = true+}++// ChunkOffset returns the byte position of this result's secret within the chunk data.+func (r *Result) ChunkOffset() int64 {+ return r.chunkOffset+}++// HasChunkOffset reports whether a chunk offset has been explicitly set on this result.+func (r *Result) HasChunkOffset() bool {+ return r.chunkOffsetSet+}+// redactSecrets replaces all instances of the given secrets with [REDACTED] in the error message.func redactSecrets(err error, secrets ...string) error {lastErr := unwrapToLast(err)diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 335855a72..cd4d2b4bd 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -1186,6 +1186,8 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {results = e.filterResults(ctx, data.detector, results)}+ AssignDuplicateLineOffsets(&data.chunk, results)+for _, res := range results {e.processResult(ctx, res, data.chunk, data.decoder, data.detector.Description(), isFalsePositive)}@@ -1339,21 +1341,39 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {}}-// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.-func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {- // get the primary secret value from the result if set- secret := result.GetPrimarySecretValue()+// effectiveSecret returns the canonical secret string for a result, preferring+// the primary secret value and falling back to Raw. Callers that compute or+// consume chunk offsets must go through this helper so the two stay in sync.+func effectiveSecret(r *detectors.Result) string {+ secret := r.GetPrimarySecretValue()if secret == "" {- secret = string(result.Raw)+ secret = string(r.Raw)}+ return secret+}- before, after, found := bytes.Cut(chunk.Data, []byte(secret))- if !found {- return 0, false+// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.+func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {+ secretBytes := []byte(effectiveSecret(result))++ // Locate the byte offset of the secret in chunk.Data. If a chunk offset was+ // pre-assigned (for duplicate secrets), use it directly to find the correct+ // occurrence instead of always matching the first one.+ var offset int+ if result.HasChunkOffset() {+ offset = int(result.ChunkOffset())+ } else {+ offset = bytes.Index(chunk.Data, secretBytes)+ if offset == -1 {+ return 0, false+ }}- lineNumber := int64(bytes.Count(before, []byte("\n")))++ lineNumber := int64(bytes.Count(chunk.Data[:offset], []byte("\n")))result.SetPrimarySecretLine(lineNumber)- // If the line contains the ignore tag, we should ignore the result.++ // If the line containing the secret has the ignore tag, we should ignore the result.+ after := chunk.Data[offset+len(secretBytes):]endLine := bytes.Index(after, []byte("\n"))if endLine == -1 {endLine = len(after)@@ -1364,6 +1384,46 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,return lineNumber, false}+// AssignDuplicateLineOffsets pre-computes byte offsets for results that share the same+// secret value within a chunk. This allows FragmentLineOffset to locate the correct+// occurrence instead of always finding the first one.+func AssignDuplicateLineOffsets(chunk *sources.Chunk, results []detectors.Result) {+ // Group result indices by their secret value.+ type group struct {+ secret string+ indices []int+ }+ seen := make(map[string]int) // secret -> index into groups slice+ var groups []group++ for i := range results {+ secret := effectiveSecret(&results[i])+ if idx, ok := seen[secret]; ok {+ groups[idx].indices = append(groups[idx].indices, i)+ } else {+ seen[secret] = len(groups)+ groups = append(groups, group{secret: secret, indices: []int{i}})+ }+ }++ for _, g := range groups {+ if len(g.indices) <= 1 {+ continue+ }+ secretBytes := []byte(g.secret)+ searchStart := 0+ for _, ri := range g.indices {+ pos := bytes.Index(chunk.Data[searchStart:], secretBytes)+ if pos == -1 {+ break+ }+ absPos := searchStart + pos+ results[ri].SetChunkOffset(int64(absPos))+ searchStart = absPos + len(secretBytes)+ }+ }+}+// FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata.// It returns:// - The first line number of the fragment.diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex d68960096..7f1ba8d4a 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -231,6 +231,93 @@ func TestFragmentLineOffsetWithPrimarySecretMultiline(t *testing.T) {assert.Equal(t, int64(2), lineOffset)}+// TestFragmentLineOffset_DuplicateSecrets verifies that when the same secret+// appears on multiple lines within a chunk, each result receives the correct+// line number rather than always reporting the first occurrence's line.+// Regression test for https://github.com/trufflesecurity/trufflehog/issues/2502+func TestFragmentLineOffset_DuplicateSecrets(t *testing.T) {+ secret := []byte("AKIA1234567890ABCDEF")+ chunk := &sources.Chunk{+ Data: []byte("line1\n" + // line 0+ "line2\n" + // line 1+ "AKIA1234567890ABCDEF\n" + // line 2 (first occurrence)+ "line4\n" + // line 3+ "AKIA1234567890ABCDEF\n" + // line 4 (second occurrence)+ "line6\n" + // line 5+ "AKIA1234567890ABCDEF\n"), // line 6 (third occurrence)+ }++ results := []detectors.Result{+ {Raw: secret},+ {Raw: secret},+ {Raw: secret},+ }+ expectedLines := []int64{2, 4, 6}++ AssignDuplicateLineOffsets(chunk, results)++ seen := make(map[int64]bool)+ for i, res := range results {+ lineOffset, _ := FragmentLineOffset(chunk, &res)+ assert.Equal(t, expectedLines[i], lineOffset,+ "result[%d]: expected line %d but got %d", i, expectedLines[i], lineOffset)+ assert.False(t, seen[lineOffset],+ "result[%d]: line %d was already reported by a previous result (duplicate line number)", i, lineOffset)+ seen[lineOffset] = true+ }+}++func TestAssignDuplicateLineOffsets(t *testing.T) {+ chunk := &sources.Chunk{+ Data: []byte("aaa\nbbb\naaa\nccc\naaa\n"),+ }+ results := []detectors.Result{+ {Raw: []byte("aaa")},+ {Raw: []byte("aaa")},+ {Raw: []byte("aaa")},+ {Raw: []byte("bbb")},+ }+ AssignDuplicateLineOffsets(chunk, results)++ // Duplicates get offsets assigned.+ assert.True(t, results[0].HasChunkOffset())+ assert.Equal(t, int64(0), results[0].ChunkOffset())++ assert.True(t, results[1].HasChunkOffset())+ assert.Equal(t, int64(8), results[1].ChunkOffset()) // "aaa\nbbb\n" = 8 bytes++ assert.True(t, results[2].HasChunkOffset())+ assert.Equal(t, int64(16), results[2].ChunkOffset()) // "aaa\nbbb\naaa\nccc\n" = 16 bytes++ // Unique secret does not get an offset.+ assert.False(t, results[3].HasChunkOffset())+}++func TestFragmentLineOffset_DuplicateSecretsWithIgnoreTag(t *testing.T) {+ secret := []byte("mysecret")+ chunk := &sources.Chunk{+ Data: []byte("mysecret\nfoo\nmysecret trufflehog:ignore\nbar\nmysecret\n"),+ }+ results := []detectors.Result{+ {Raw: secret},+ {Raw: secret},+ {Raw: secret},+ }+ AssignDuplicateLineOffsets(chunk, results)++ line0, ignored0 := FragmentLineOffset(chunk, &results[0])+ assert.Equal(t, int64(0), line0)+ assert.False(t, ignored0)++ line1, ignored1 := FragmentLineOffset(chunk, &results[1])+ assert.Equal(t, int64(2), line1)+ assert.True(t, ignored1)++ line2, ignored2 := FragmentLineOffset(chunk, &results[2])+ assert.Equal(t, int64(4), line2)+ assert.False(t, ignored2)+}+func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) {data := make([]byte, 0, 4096)needle := []byte("needle")