Skip to content

Find, verify, and analyze leaked credentials

refactor: use HandleFile for Jenkins build log processing to improve chunking (#4225)

ahrav6641d4ba5b68
diff --git a/pkg/sources/jenkins/jenkins.go b/pkg/sources/jenkins/jenkins.go
index b84274583..88a4b627d 100644
--- a/pkg/sources/jenkins/jenkins.go
+++ b/pkg/sources/jenkins/jenkins.go
@@ -11,12 +11,13 @@ import (
"time"
"github.com/go-errors/errors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/log"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/roundtripper"
@@ -411,13 +412,7 @@ func (s *Source) chunkBuild(
buildLogURL.String())
}
- buildLog, err := io.ReadAll(resp.Body)
- if err != nil {
- return fmt.Errorf("error reading build log response body from %q: %w", buildLogURL.String(), err)
- }
-
- ctx.Logger().V(4).Info("scanning build log")
- chunksChan <- &sources.Chunk{
+ chunkSkel := &sources.Chunk{
SourceName: s.name,
SourceID: s.SourceID(),
SourceType: s.Type(),
@@ -431,11 +426,11 @@ func (s *Source) chunkBuild(
},
},
},
- Data: buildLog,
Verify: s.verify,
}
- return nil
+ ctx.Logger().V(4).Info("scanning build log")
+ return handlers.HandleFile(ctx, resp.Body, chunkSkel, sources.ChanReporter{Ch: chunksChan})
}
type JenkinsJobResponse struct {
diff --git a/pkg/sources/jenkins/jenkins_test.go b/pkg/sources/jenkins/jenkins_test.go
new file mode 100644
index 000000000..359baa5e8
--- /dev/null
+++ b/pkg/sources/jenkins/jenkins_test.go
@@ -0,0 +1,299 @@
+package jenkins
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/types/known/anypb"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const (
+ KB = 1024
+ MB = 1024 * KB
+)
+
+// generateTestData creates a string of exactly the specified size using the given pattern.
+func generateTestData(size int, pattern string) string {
+ if len(pattern) == 0 {
+ pattern = "X" // fallback pattern
+ }
+
+ var builder strings.Builder
+ builder.Grow(size)
+
+ for builder.Len() < size {
+ remaining := size - builder.Len()
+ if remaining >= len(pattern) {
+ builder.WriteString(pattern)
+ } else {
+ // Truncate the pattern to fill exactly the remaining bytes.
+ builder.WriteString(pattern[:remaining])
+ }
+ }
+
+ return builder.String()
+}
+
+// createMockJenkinsServer creates a test HTTP server that simulates Jenkins API responses.
+func createMockJenkinsServer(jobName string, buildNumber int, logContent string) *httptest.Server {
+ mux := http.NewServeMux()
+ server := httptest.NewServer(mux)
+
+ // Mock the main Jenkins API endpoint that lists jobs.
+ mux.HandleFunc("/api/json", func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.RawQuery, "tree=jobs") {
+ w.Header().Set("Content-Type", "application/json")
+ response := fmt.Sprintf(
+ `{"jobs":[{"_class":"org.jenkinsci.plugins.workflow.job.WorkflowJob",`+
+ `"name":"%s","url":"%s/job/%s/"}]}`, jobName, server.URL, jobName)
+ fmt.Fprint(w, response)
+ } else {
+ w.Header().Set("Content-Type", "application/json")
+ fmt.Fprint(w, `{"jobs":[]}`)
+ }
+ })
+
+ // Mock the job-specific API endpoint that lists builds for a particular job.
+ mux.HandleFunc(fmt.Sprintf("/job/%s/api/json", jobName), func(w http.ResponseWriter, r *http.Request) {
+ if strings.Contains(r.URL.RawQuery, "tree=builds") {
+ w.Header().Set("Content-Type", "application/json")
+ response := fmt.Sprintf(
+ `{"builds":[{"number":%d,"url":"%s/job/%s/%d/"}]}`, buildNumber, server.URL, jobName, buildNumber)
+ fmt.Fprint(w, response)
+ } else {
+ w.Header().Set("Content-Type", "application/json")
+ fmt.Fprint(w, `{"builds":[]}`)
+ }
+ })
+
+ // Mock the console text endpoint that returns the actual build log content.
+ // This is where the test data payload is served to verify chunking behavior.
+ mux.HandleFunc(fmt.Sprintf("/job/%s/%d/consoleText", jobName, buildNumber), func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain")
+ fmt.Fprint(w, logContent)
+ })
+
+ return server
+}
+
+// TestJenkinsVariousSizes verifies that Jenkins build logs are properly chunked
+// across different data sizes that represent real-world scenarios from small
+// logs to large CI/CD outputs.
+func TestJenkinsVariousSizes(t *testing.T) {
+ testCases := []struct {
+ name string
+ dataSize int
+ pattern string
+ jobName string
+ buildNumber int
+ }{
+ {
+ name: "small_60KB",
+ dataSize: 60 * KB,
+ pattern: "This is a line in the build log with some sensitive data\n",
+ jobName: "test-job",
+ buildNumber: 42,
+ },
+ {
+ name: "large_1MB",
+ dataSize: 1 * MB,
+ pattern: "Line with potential secrets like api_key=abc123def456\n",
+ jobName: "large-job",
+ buildNumber: 1,
+ },
+ {
+ name: "medium_80KB",
+ dataSize: 80 * KB,
+ pattern: "Line with secret: api_key=sk-123abc456def\n",
+ jobName: "medium-job",
+ buildNumber: 123,
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ logContent := generateTestData(tc.dataSize, tc.pattern)
+ t.Logf("Generated %d bytes (%.2f KB) of test data", tc.dataSize, float64(tc.dataSize)/float64(KB))
+
+ server := createMockJenkinsServer(tc.jobName, tc.buildNumber, logContent)
+ defer server.Close()
+
+ s := new(Source)
+ conn, err := anypb.New(&sourcespb.Jenkins{
+ Endpoint: server.URL,
+ Credential: &sourcespb.Jenkins_BasicAuth{
+ BasicAuth: &credentialspb.BasicAuth{
+ Username: "testuser",
+ Password: "testpass",
+ },
+ },
+ })
+ require.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ err = s.Init(ctx, "test-jenkins-"+tc.name, 0, 1, false, conn, runtime.NumCPU())
+ require.NoError(t, err)
+
+ jobs, err := s.GetJenkinsJobs(ctx)
+ require.NoError(t, err)
+ require.NotEmpty(t, jobs.Jobs, "No jobs found. This indicates a mock server setup issue.")
+
+ chunksChan := make(chan *sources.Chunk, 200)
+ done := make(chan error, 1)
+
+ go func() {
+ defer close(chunksChan)
+ done <- s.Chunks(ctx, chunksChan)
+ }()
+
+ var chunks []*sources.Chunk
+ var totalDataSize int
+ maxChunkSize := 0
+ for chunk := range chunksChan {
+ chunks = append(chunks, chunk)
+ totalDataSize += len(chunk.Data)
+ if len(chunk.Data) > maxChunkSize {
+ maxChunkSize = len(chunk.Data)
+ }
+ }
+
+ require.NoError(t, <-done)
+ require.NotEmpty(t, chunks, "No chunks were received.")
+
+ // Verify that large logs are actually being split into multiple chunks.
+ // This catches regressions where chunking logic might not be working.
+ // Data larger than a single chunk should result in multiple chunks.
+ if tc.dataSize > sources.ChunkSize && len(chunks) <= 1 {
+ t.Logf("Got only %d chunk for data size %d bytes (chunk size: %d bytes), may indicate chunking not working as expected",
+ len(chunks), tc.dataSize, sources.ChunkSize)
+ }
+
+ // Ensure no individual chunk exceeds the maximum allowed size.
+ // This validates that the chunking mechanism respects size limits.
+ assert.LessOrEqual(t, maxChunkSize, sources.TotalChunkSize,
+ "Found chunk larger than expected: %d bytes (max expected %d bytes)",
+ maxChunkSize, sources.TotalChunkSize)
+
+ // Validate data integrity by checking that total output matches input size.
+ // Lower bound ensures no data loss; upper bound catches excessive duplication
+ // from overlapping peek data between adjacent chunks.
+ assert.GreaterOrEqual(t, totalDataSize, tc.dataSize,
+ "Total data size %d is less than original %d - suggests data loss",
+ totalDataSize, tc.dataSize)
+ assert.LessOrEqual(t, totalDataSize, tc.dataSize*3,
+ "Total data size %d is much larger than original %d - suggests excessive duplication",
+ totalDataSize, tc.dataSize)
+
+ chunk := chunks[0]
+ assert.Equal(t, "test-jenkins-"+tc.name, chunk.SourceName)
+
+ jenkinsMetadata := chunk.SourceMetadata.GetJenkins()
+ require.NotNil(t, jenkinsMetadata, "Missing Jenkins metadata")
+ assert.Equal(t, tc.jobName, jenkinsMetadata.ProjectName)
+ assert.Equal(t, int64(tc.buildNumber), jenkinsMetadata.BuildNumber)
+
+ expectedLink := fmt.Sprintf("%s/job/%s/%d/consoleText", server.URL, tc.jobName, tc.buildNumber)
+ assert.Equal(t, expectedLink, jenkinsMetadata.Link)
+ })
+ }
+}
+
+// TestJenkinsChunkBuildDirect tests the chunkBuild method in isolation to verify
+// that build log chunking works correctly without the overhead of the full source
+// initialization and job discovery process.
+func TestJenkinsChunkBuildDirect(t *testing.T) {
+ // Use a size that will definitely require chunking to test the splitting logic.
+ largeLogContent := generateTestData(500*KB, "Line with secret: api_key=sk-123abc456def\n")
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if strings.HasSuffix(r.URL.Path, "/consoleText") {
+ w.Header().Set("Content-Type", "text/plain")
+ fmt.Fprint(w, largeLogContent)
+ } else {
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ s := new(Source)
+ conn, err := anypb.New(&sourcespb.Jenkins{
+ Endpoint: server.URL,
+ Credential: &sourcespb.Jenkins_BasicAuth{
+ BasicAuth: &credentialspb.BasicAuth{Username: "test", Password: "test"},
+ },
+ })
+ require.NoError(t, err)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ err = s.Init(ctx, "test-chunk-build", 0, 1, false, conn, runtime.NumCPU())
+ require.NoError(t, err)
+
+ mockBuild := JenkinsBuild{
+ Number: 123,
+ Url: server.URL + "/job/test-project/123/",
+ }
+
+ chunksChan := make(chan *sources.Chunk, 200)
+
+ go func() {
+ defer close(chunksChan)
+ err := s.chunkBuild(ctx, mockBuild, "test-project", chunksChan)
+ assert.NoError(t, err)
+ }()
+
+ var chunks []*sources.Chunk
+ var totalDataSize int
+ maxChunkSize := 0
+ for chunk := range chunksChan {
+ chunks = append(chunks, chunk)
+ totalDataSize += len(chunk.Data)
+ if len(chunk.Data) > maxChunkSize {
+ maxChunkSize = len(chunk.Data)
+ }
+ }
+
+ require.NotEmpty(t, chunks, "No chunks were received from chunkBuild.")
+
+ assert.LessOrEqual(t, maxChunkSize, sources.TotalChunkSize,
+ "Found chunk larger than expected: %d bytes (max expected %d bytes)",
+ maxChunkSize, sources.TotalChunkSize)
+
+ // Ensure that direct chunking maintains data integrity with the same
+ // bounds checking as the full integration test.
+ originalSize := len(largeLogContent)
+ assert.GreaterOrEqual(t, totalDataSize, originalSize,
+ "Total data size %d is less than original %d - suggests data loss",
+ totalDataSize, originalSize)
+ assert.LessOrEqual(t, totalDataSize, originalSize*3,
+ "Total data size %d is much larger than original %d - suggests excessive duplication",
+ totalDataSize, originalSize)
+
+ chunk := chunks[0]
+ assert.Equal(t, "test-chunk-build", chunk.SourceName)
+
+ jenkinsMetadata := chunk.SourceMetadata.GetJenkins()
+ require.NotNil(t, jenkinsMetadata, "Missing Jenkins metadata")
+ assert.Equal(t, "test-project", jenkinsMetadata.ProjectName)
+ assert.Equal(t, int64(123), jenkinsMetadata.BuildNumber)
+
+ expectedLink := server.URL + "/job/test-project/123/consoleText"
+ assert.Equal(t, expectedLink, jenkinsMetadata.Link)
+}