Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[cleanup] - Extract buffer logic (#2409)
ahrav40bbab8add8a
diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.goindex dbcf4604c..3c364644a 100644--- a/pkg/gitparse/gitparse.go+++ b/pkg/gitparse/gitparse.go@@ -16,6 +16,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ bufferwriter "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffer_writer"bufferedfilewriter "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffered_file_writer")@@ -47,55 +48,6 @@ type contentWriter interface { // Write appends data to the content storage.String() (string, error)}-// state represents the current mode of buffer.-type state uint8--const (- // writeOnly indicates the buffer is in write-only mode.- writeOnly state = iota- // readOnly indicates the buffer has been closed and is in read-only mode.- readOnly-)--// buffer is a wrapper around bytes.Buffer, implementing the contentWriter interface.-// This allows bytes.Buffer to be used wherever a contentWriter is required, ensuring compatibility-// with the contentWriter interface while leveraging the existing implementation of bytes.Buffer.-type buffer struct {- state state // current state of the buffer (writeOnly or readOnly)- bytes.Buffer-}--func newBuffer() *buffer { return &buffer{state: writeOnly} }--// Write delegates the writing operation to the underlying bytes.Buffer, ignoring the context.-// The context is included to satisfy the contentWriter interface, allowing for future extensions-// where context handling might be necessary (e.g., for timeouts or cancellation).-func (b *buffer) Write(_ context.Context, data []byte) (int, error) {- if b.state == readOnly {- return 0, fmt.Errorf("buffer is in read-only mode")- }- return b.Buffer.Write(data)-}--// ReadCloser provides a read-closer for the buffer's content.-// It wraps the buffer's content in a NopCloser to provide a ReadCloser without additional closing behavior,-// as closing a bytes.Buffer is a no-op.-func (b *buffer) ReadCloser() (io.ReadCloser, error) {- if b.state == writeOnly {- return nil, fmt.Errorf("buffer is in write-only mode")- }- return io.NopCloser(bytes.NewReader(b.Bytes())), nil-}--// CloseForWriting is a no-op for buffer, as there is no resource cleanup needed for bytes.Buffer.-func (b *buffer) CloseForWriting() error {- b.state = readOnly- return nil-}--// String returns the buffer's content as a string.-func (b *buffer) String() (string, error) { return b.Buffer.String(), nil }-// Diff contains the information about a file diff in a commit.// It abstracts the underlying content representation, allowing for flexible handling of diff content.// The use of contentWriter enables the management of diff data either in memory or on disk,@@ -124,8 +76,8 @@ func withCustomContentWriter(cr contentWriter) diffOption {// All Diffs must have an associated commit.// The contentWriter is used to manage the diff's content, allowing for flexible handling of diff data.// By default, a buffer is used as the contentWriter, but this can be overridden with a custom contentWriter.-func newDiff(commit *Commit, opts ...diffOption) *Diff {- diff := &Diff{Commit: commit, contentWriter: newBuffer()}+func newDiff(ctx context.Context, commit *Commit, opts ...diffOption) *Diff {+ diff := &Diff{Commit: commit, contentWriter: bufferwriter.New(ctx)}for _, opt := range opts {opt(diff)}@@ -356,13 +308,13 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, diffChan chanvar latestState = Initialdiff := func(c *Commit, opts ...diffOption) *Diff {- opts = append(opts, withCustomContentWriter(newBuffer()))- return newDiff(c, opts...)+ opts = append(opts, withCustomContentWriter(bufferwriter.New(ctx)))+ return newDiff(ctx, c, opts...)}if c.useCustomContentWriter {diff = func(c *Commit, opts ...diffOption) *Diff {- opts = append(opts, withCustomContentWriter(bufferedfilewriter.New()))- return newDiff(c, opts...)+ opts = append(opts, withCustomContentWriter(bufferedfilewriter.New(ctx)))+ return newDiff(ctx, c, opts...)}}currentDiff := diff(currentCommit)diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.goindex df2cfbfd2..293973fc3 100644--- a/pkg/gitparse/gitparse_test.go+++ b/pkg/gitparse/gitparse_test.go@@ -6,9 +6,8 @@ import ("testing""time"- "github.com/stretchr/testify/assert"-"github.com/trufflesecurity/trufflehog/v3/pkg/context"+ bufferwriter "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffer_writer"bufferedfilewriter "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffered_file_writer")@@ -674,18 +673,20 @@ func TestCommitParsing(t *testing.T) {}func newBufferedFileWriterWithContent(content []byte) *bufferedfilewriter.BufferedFileWriter {- b := bufferedfilewriter.New()- _, err := b.Write(context.Background(), content) // Using Write method to add content+ ctx := context.Background()+ b := bufferedfilewriter.New(ctx)+ _, err := b.Write(ctx, content) // Using Write method to add contentif err != nil {panic(err)}return b}-func newBufferWithContent(content []byte) *buffer {- var b buffer- _, _ = b.Write(context.Background(), content) // Using Write method to add content- return &b+func newBufferWithContent(content []byte) *bufferwriter.BufferWriter {+ ctx := context.Background()+ b := bufferwriter.New(ctx)+ _, _ = b.Write(ctx, content) // Using Write method to add content+ return b}func TestStagedDiffParsing(t *testing.T) {@@ -2292,32 +2293,3 @@ index 2ee133b..12b4843 100644+output = json+region = us-east-2`--func TestBufferWriterStateTransitionOnClose(t *testing.T) {- t.Parallel()- writer := new(buffer)-- // Initially, the writer should be in write-only mode.- assert.Equal(t, writeOnly, writer.state)-- // Perform some write operation.- _, err := writer.Write(context.Background(), []byte("test data"))- assert.NoError(t, err)-- // Close the writer.- err = writer.CloseForWriting()- assert.NoError(t, err)-- // After closing, the writer should be in read-only mode.- assert.Equal(t, readOnly, writer.state)-}--func TestBufferWriterWriteInReadOnlyState(t *testing.T) {- t.Parallel()- writer := new(buffer)- _ = writer.CloseForWriting() // Transition to read-only mode-- // Attempt to write in read-only mode.- _, err := writer.Write(context.Background(), []byte("should fail"))- assert.Error(t, err)-}diff --git a/pkg/writers/buffer/buffer.go b/pkg/writers/buffer/buffer.gonew file mode 100644index 000000000..9adea74cb--- /dev/null+++ b/pkg/writers/buffer/buffer.go@@ -0,0 +1,183 @@+// Package buffer provides a custom buffer type that includes metrics for tracking buffer usage.+// It also provides a pool for managing buffer reusability.+package buffer++import (+ "bytes"+ "fmt"+ "sync"+ "time"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++type poolMetrics struct{}++func (poolMetrics) recordShrink(amount int) {+ shrinkCount.Inc()+ shrinkAmount.Add(float64(amount))+}++func (poolMetrics) recordBufferRetrival() {+ activeBufferCount.Inc()+ checkoutCount.Inc()+ bufferCount.Inc()+}++func (poolMetrics) recordBufferReturn(bufCap, bufLen int64) {+ activeBufferCount.Dec()+ totalBufferSize.Add(float64(bufCap))+ totalBufferLength.Add(float64(bufLen))+}++// PoolOpts is a function that configures a BufferPool.+type PoolOpts func(pool *Pool)++// Pool of buffers.+type Pool struct {+ *sync.Pool+ bufferSize uint32++ metrics poolMetrics+}++const defaultBufferSize = 1 << 12 // 4KB+// NewBufferPool creates a new instance of BufferPool.+func NewBufferPool(opts ...PoolOpts) *Pool {+ pool := &Pool{bufferSize: defaultBufferSize}++ for _, opt := range opts {+ opt(pool)+ }+ pool.Pool = &sync.Pool{+ New: func() any {+ return &Buffer{Buffer: bytes.NewBuffer(make([]byte, 0, pool.bufferSize))}+ },+ }++ return pool+}++// Get returns a Buffer from the pool.+func (p *Pool) Get(ctx context.Context) *Buffer {+ buf, ok := p.Pool.Get().(*Buffer)+ if !ok {+ ctx.Logger().Error(fmt.Errorf("Buffer pool returned unexpected type"), "using new Buffer")+ buf = &Buffer{Buffer: bytes.NewBuffer(make([]byte, 0, p.bufferSize))}+ }+ p.metrics.recordBufferRetrival()+ buf.resetMetric()++ return buf+}++// Put returns a Buffer to the pool.+func (p *Pool) Put(buf *Buffer) {+ p.metrics.recordBufferReturn(int64(buf.Cap()), int64(buf.Len()))++ // If the Buffer is more than twice the default size, replace it with a new Buffer.+ // This prevents us from returning very large buffers to the pool.+ const maxAllowedCapacity = 2 * defaultBufferSize+ if buf.Cap() > maxAllowedCapacity {+ p.metrics.recordShrink(buf.Cap() - defaultBufferSize)+ buf = &Buffer{Buffer: bytes.NewBuffer(make([]byte, 0, p.bufferSize))}+ } else {+ // Reset the Buffer to clear any existing data.+ buf.Reset()+ }+ buf.recordMetric()++ p.Pool.Put(buf)+}++// Buffer is a wrapper around bytes.Buffer that includes a timestamp for tracking Buffer checkout duration.+type Buffer struct {+ *bytes.Buffer+ checkedOutAt time.Time+}++// NewBuffer creates a new instance of Buffer.+func NewBuffer() *Buffer { return &Buffer{Buffer: bytes.NewBuffer(make([]byte, 0, defaultBufferSize))} }++func (b *Buffer) Grow(size int) {+ b.Buffer.Grow(size)+ b.recordGrowth(size)+}++func (b *Buffer) resetMetric() { b.checkedOutAt = time.Now() }++func (b *Buffer) recordMetric() {+ dur := time.Since(b.checkedOutAt)+ checkoutDuration.Observe(float64(dur.Microseconds()))+ checkoutDurationTotal.Add(float64(dur.Microseconds()))+}++func (b *Buffer) recordGrowth(size int) {+ growCount.Inc()+ growAmount.Add(float64(size))+}++// Write date to the buffer.+func (b *Buffer) Write(ctx context.Context, data []byte) (int, error) {+ if b.Buffer == nil {+ // This case should ideally never occur if buffers are properly managed.+ ctx.Logger().Error(fmt.Errorf("buffer is nil, initializing a new buffer"), "action", "initializing_new_buffer")+ b.Buffer = bytes.NewBuffer(make([]byte, 0, defaultBufferSize))+ b.resetMetric()+ }++ size := len(data)+ bufferLength := b.Buffer.Len()+ totalSizeNeeded := bufferLength + size+ // If the total size is within the threshold, write to the buffer.+ ctx.Logger().V(4).Info(+ "writing to buffer",+ "data_size", size,+ "content_size", bufferLength,+ )++ availableSpace := b.Buffer.Cap() - bufferLength+ growSize := totalSizeNeeded - bufferLength+ if growSize > availableSpace {+ ctx.Logger().V(4).Info(+ "buffer size exceeded, growing buffer",+ "current_size", bufferLength,+ "new_size", totalSizeNeeded,+ "available_space", availableSpace,+ "grow_size", growSize,+ )+ // We are manually growing the buffer so we can track the growth via metrics.+ // Knowing the exact data size, we directly resize to fit it, rather than exponential growth+ // which may require multiple allocations and copies if the size required is much larger+ // than double the capacity. Our approach aligns with default behavior when growth sizes+ // happen to match current capacity, retaining asymptotic efficiency benefits.+ b.Buffer.Grow(growSize)+ }++ return b.Buffer.Write(data)+}++// readCloser is a custom implementation of io.ReadCloser. It wraps a bytes.Reader+// for reading data from an in-memory buffer and includes an onClose callback.+// The onClose callback is used to return the buffer to the pool, ensuring buffer re-usability.+type readCloser struct {+ *bytes.Reader+ onClose func()+}++// ReadCloser creates a new instance of readCloser.+func ReadCloser(data []byte, onClose func()) *readCloser {+ return &readCloser{Reader: bytes.NewReader(data), onClose: onClose}+}++// Close implements the io.Closer interface. It calls the onClose callback to return the buffer+// to the pool, enabling buffer reuse. This method should be called by the consumers of ReadCloser+// once they have finished reading the data to ensure proper resource management.+func (brc *readCloser) Close() error {+ if brc.onClose == nil {+ return nil+ }++ brc.onClose() // Return the buffer to the pool+ return nil+}diff --git a/pkg/writers/buffer/buffer_test.go b/pkg/writers/buffer/buffer_test.gonew file mode 100644index 000000000..da40e6ce0--- /dev/null+++ b/pkg/writers/buffer/buffer_test.go@@ -0,0 +1,167 @@+package buffer++import (+ "bytes"+ "testing"++ "github.com/stretchr/testify/assert"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++func TestNewBufferPool(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ opts []PoolOpts+ expectedBuffSize uint32+ }{+ {name: "Default pool size", expectedBuffSize: defaultBufferSize},+ {+ name: "Custom pool size",+ opts: []PoolOpts{func(p *Pool) { p.bufferSize = 8 * 1024 }}, // 8KB+ expectedBuffSize: 8 * 1024,+ },+ }++ for _, tc := range tests {+ tc := tc+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ pool := NewBufferPool(tc.opts...)+ assert.Equal(t, tc.expectedBuffSize, pool.bufferSize)+ })+ }+}++func TestBufferPoolGetPut(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ preparePool func(p *Pool) *Buffer // Prepare the pool and return an initial buffer to put if needed+ expectedCapBefore int // Expected capacity before putting it back+ expectedCapAfter int // Expected capacity after retrieving it again+ }{+ {+ name: "Get new buffer and put back without modification",+ preparePool: func(_ *Pool) *Buffer {+ return nil // No initial buffer to put+ },+ expectedCapBefore: int(defaultBufferSize),+ expectedCapAfter: int(defaultBufferSize),+ },+ {+ name: "Put oversized buffer, expect shrink",+ preparePool: func(p *Pool) *Buffer {+ buf := &Buffer{Buffer: bytes.NewBuffer(make([]byte, 0, 3*defaultBufferSize))}+ return buf+ },+ expectedCapBefore: int(defaultBufferSize),+ expectedCapAfter: int(defaultBufferSize), // Should shrink back to default+ },+ }++ for _, tc := range tests {+ tc := tc+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ pool := NewBufferPool()+ initialBuf := tc.preparePool(pool)+ if initialBuf != nil {+ pool.Put(initialBuf)+ }++ buf := pool.Get(context.Background())+ assert.Equal(t, tc.expectedCapBefore, buf.Cap())++ pool.Put(buf)++ bufAfter := pool.Get(context.Background())+ assert.Equal(t, tc.expectedCapAfter, bufAfter.Cap())+ })+ }+}++func TestBufferWrite(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ initialCapacity int+ writeDataSequence [][]byte // Sequence of writes to simulate multiple writes+ expectedSize int+ expectedCap int+ }{+ {+ name: "Write to empty buffer",+ initialCapacity: defaultBufferSize,+ writeDataSequence: [][]byte{+ []byte("hello"),+ },+ expectedSize: 5,+ expectedCap: defaultBufferSize, // No growth for small data+ },+ {+ name: "Write causing growth",+ initialCapacity: 10, // Small initial capacity to force growth+ writeDataSequence: [][]byte{+ []byte("this is a longer string exceeding initial capacity"),+ },+ expectedSize: 50,+ expectedCap: 50,+ },+ {+ name: "Write nil data",+ initialCapacity: defaultBufferSize,+ writeDataSequence: [][]byte{nil},+ expectedCap: defaultBufferSize,+ },+ {+ name: "Repeated writes, cumulative growth",+ initialCapacity: 20, // Set an initial capacity to test growth over multiple writes+ writeDataSequence: [][]byte{+ []byte("first write, "),+ []byte("second write, "),+ []byte("third write exceeding the initial capacity."),+ },+ expectedSize: 70,+ expectedCap: 70, // Expect capacity to grow to accommodate all writes+ },+ {+ name: "Write large single data to test significant growth",+ initialCapacity: 50, // Set an initial capacity smaller than the data to be written+ writeDataSequence: [][]byte{+ bytes.Repeat([]byte("a"), 1024), // 1KB data to significantly exceed initial capacity+ },+ expectedSize: 1024,+ expectedCap: 1024,+ },+ }++ for _, tc := range tests {+ tc := tc+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ buf := &Buffer{Buffer: bytes.NewBuffer(make([]byte, 0, tc.initialCapacity))}+ totalWritten := 0+ for _, data := range tc.writeDataSequence {+ n, err := buf.Write(context.Background(), data)+ assert.NoError(t, err)++ totalWritten += n+ }+ assert.Equal(t, tc.expectedSize, totalWritten)+ assert.Equal(t, tc.expectedSize, buf.Len())+ assert.GreaterOrEqual(t, buf.Cap(), tc.expectedCap)+ })+ }+}++func TestReadCloserClose(t *testing.T) {+ t.Parallel()+ onCloseCalled := false+ rc := ReadCloser([]byte("data"), func() { onCloseCalled = true })++ err := rc.Close()+ assert.NoError(t, err)+ assert.True(t, onCloseCalled, "onClose callback should be called upon Close")+}diff --git a/pkg/writers/buffer/metrics.go b/pkg/writers/buffer/metrics.gonew file mode 100644index 000000000..2d4d8afce--- /dev/null+++ b/pkg/writers/buffer/metrics.go@@ -0,0 +1,88 @@+package buffer++import (+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+)++var (+ activeBufferCount = promauto.NewGauge(prometheus.GaugeOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "active_buffer_count",+ Help: "Current number of active buffers.",+ })++ bufferCount = promauto.NewGauge(prometheus.GaugeOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "buffer_count",+ Help: "Total number of buffers managed by the pool.",+ })++ totalBufferLength = promauto.NewGauge(prometheus.GaugeOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "total_buffer_length",+ Help: "Total length of all buffers combined.",+ })++ totalBufferSize = promauto.NewGauge(prometheus.GaugeOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "total_buffer_size",+ Help: "Total size of all buffers combined.",+ })++ growCount = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "grow_count",+ Help: "Total number of times buffers in the pool have grown.",+ })++ growAmount = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "grow_amount",+ Help: "Total amount of bytes buffers in the pool have grown by.",+ })++ shrinkCount = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "shrink_count",+ Help: "Total number of times buffers in the pool have shrunk.",+ })++ shrinkAmount = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "shrink_amount",+ Help: "Total amount of bytes buffers in the pool have shrunk by.",+ })++ checkoutDurationTotal = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "checkout_duration_total_us",+ Help: "Total duration in microseconds of Buffer checkouts.",+ })++ checkoutDuration = promauto.NewHistogram(prometheus.HistogramOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "checkout_duration_us",+ Help: "Duration in microseconds of Buffer checkouts.",+ Buckets: []float64{50, 500, 5000},+ })++ checkoutCount = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "checkout_count",+ Help: "Total number of Buffer checkouts.",+ })+)diff --git a/pkg/writers/buffer_writer/bufferwriter.go b/pkg/writers/buffer_writer/bufferwriter.gonew file mode 100644index 000000000..5cc70490e--- /dev/null+++ b/pkg/writers/buffer_writer/bufferwriter.go@@ -0,0 +1,106 @@+// Package bufferwritter provides a contentWriter implementation using a shared buffer pool for memory management.+package bufferwriter++import (+ "fmt"+ "io"+ "time"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffer"+)++type metrics struct{}++func (metrics) recordDataProcessed(size uint64, dur time.Duration) {+ totalWriteSize.Add(float64(size))+ totalWriteDuration.Add(float64(dur.Microseconds()))+}++func init() { bufferPool = buffer.NewBufferPool() }++// bufferPool is the shared Buffer pool used by all BufferedFileWriters.+// This allows for efficient reuse of buffers across multiple writers.+var bufferPool *buffer.Pool++// state represents the current mode of buffer.+type state uint8++const (+ // writeOnly indicates the buffer is in write-only mode.+ writeOnly state = iota+ // readOnly indicates the buffer has been closed and is in read-only mode.+ readOnly+)++// BufferWriter implements contentWriter, using a shared buffer pool for memory management.+type BufferWriter struct {+ buf *buffer.Buffer // The current buffer in use.+ bufPool *buffer.Pool // The buffer pool used to manage the buffer.+ size int // The total size of the content written to the buffer.+ state state // The current state of the buffer.++ metrics metrics+}++// New creates a new instance of BufferWriter.+func New(ctx context.Context) *BufferWriter {+ buf := bufferPool.Get(ctx)+ if buf == nil {+ buf = buffer.NewBuffer()+ }+ return &BufferWriter{buf: buf, state: writeOnly, bufPool: bufferPool}+}++// Write delegates the writing operation to the underlying bytes.Buffer, ignoring the context.+// The context is included to satisfy the contentWriter interface, allowing for future extensions+// where context handling might be necessary (e.g., for timeouts or cancellation).+func (b *BufferWriter) Write(ctx context.Context, data []byte) (int, error) {+ if b.state != writeOnly {+ return 0, fmt.Errorf("buffer must be in write-only mode to write data; current state: %d", b.state)+ }++ size := len(data)+ b.size += size+ start := time.Now()+ defer func(start time.Time) {+ bufferLength := uint64(b.buf.Len())+ b.metrics.recordDataProcessed(bufferLength, time.Since(start))++ ctx.Logger().V(4).Info(+ "write complete",+ "data_size", size,+ "buffer_len", bufferLength,+ "buffer_size", b.buf.Cap(),+ )+ }(start)+ return b.buf.Write(ctx, data)+}++// ReadCloser provides a read-closer for the buffer's content.+// It wraps the buffer's content in a NopCloser to provide a ReadCloser without additional closing behavior,+// as closing a bytes.Buffer is a no-op.+func (b *BufferWriter) ReadCloser() (io.ReadCloser, error) {+ if b.state != readOnly {+ return nil, fmt.Errorf("buffer is in read-only mode")+ }++ return buffer.ReadCloser(b.buf.Bytes(), func() { b.bufPool.Put(b.buf) }), nil+}++// CloseForWriting is a no-op for buffer, as there is no resource cleanup needed for bytes.Buffer.+func (b *BufferWriter) CloseForWriting() error {+ b.state = readOnly+ return nil+}++// String returns the buffer's content as a string.+func (b *BufferWriter) String() (string, error) {+ if b.buf == nil {+ return "", fmt.Errorf("buffer is nil")+ }+ return b.buf.String(), nil+}++// Len returns the length of the buffer's content.+func (b *BufferWriter) Len() int { return b.size }diff --git a/pkg/writers/buffer_writer/bufferwriter_test.go b/pkg/writers/buffer_writer/bufferwriter_test.gonew file mode 100644index 000000000..8cb8376a8--- /dev/null+++ b/pkg/writers/buffer_writer/bufferwriter_test.go@@ -0,0 +1,155 @@+package bufferwriter++import (+ "testing"++ "github.com/stretchr/testify/assert"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+)++func TestBufferWriterWrite(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ input []byte+ initialState state+ expectedError bool+ expectedSize int+ }{+ {+ name: "Write in writeOnly state with empty input",+ input: []byte(""),+ initialState: writeOnly,+ },+ {+ name: "Write in writeOnly state with non-empty input",+ input: []byte("hello"),+ initialState: writeOnly,+ expectedError: false,+ expectedSize: 5,+ },+ {+ name: "Attempt to write in readOnly state",+ input: []byte("hello"),+ initialState: readOnly,+ expectedError: true,+ },+ }++ for _, tc := range tests {+ tc := tc+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ writer := New(context.Background())+ writer.state = tc.initialState++ _, err := writer.Write(context.Background(), tc.input)+ if tc.expectedError {+ assert.Error(t, err)+ } else {+ assert.NoError(t, err)+ assert.Equal(t, tc.expectedSize, writer.Len())+ }+ })+ }+}++func TestBufferWriterReadCloser(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ initialState state+ expectedError bool+ }{+ {+ name: "Get ReadCloser in writeOnly state",+ initialState: writeOnly,+ expectedError: true,+ },+ {+ name: "Get ReadCloser in readOnly state",+ initialState: readOnly,+ },+ }++ for _, tc := range tests {+ tc := tc+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ writer := New(context.Background())+ writer.state = tc.initialState++ rc, err := writer.ReadCloser()+ if tc.expectedError {+ assert.Error(t, err)+ } else {+ assert.NoError(t, err)+ assert.NotNil(t, rc)++ // Test that the ReadCloser can be closed.+ err = rc.Close()+ assert.NoError(t, err)+ }+ })+ }+}++func TestBufferWriterCloseForWriting(t *testing.T) {+ writer := New(context.Background())+ err := writer.CloseForWriting()+ assert.NoError(t, err)+ assert.Equal(t, readOnly, writer.state)+}++func TestBufferWriterString(t *testing.T) {+ t.Parallel()+ tests := []struct {+ name string+ prepareBuffer func(*BufferWriter) // Function to prepare the buffer with data or state+ expectedStr string+ expectedError bool+ }{+ {+ name: "String with no data",+ prepareBuffer: func(_ *BufferWriter) {+ // No preparation needed, buffer is empty by default+ },+ expectedStr: "",+ expectedError: false,+ },+ {+ name: "String with data",+ prepareBuffer: func(bw *BufferWriter) {+ _, _ = bw.Write(context.Background(), []byte("test data"))+ },+ expectedStr: "test data",+ expectedError: false,+ },+ {+ name: "Buffer is nil",+ prepareBuffer: func(bw *BufferWriter) {+ bw.buf = nil+ },+ expectedStr: "",+ expectedError: true,+ },+ }++ for _, tc := range tests {+ tc := tc+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()+ writer := New(context.Background())+ tc.prepareBuffer(writer)++ result, err := writer.String()+ if tc.expectedError {+ assert.Error(t, err)+ } else {+ assert.NoError(t, err)+ assert.Equal(t, tc.expectedStr, result)+ }+ })+ }+}diff --git a/pkg/writers/buffer_writer/metrics.go b/pkg/writers/buffer_writer/metrics.gonew file mode 100644index 000000000..73a85b300--- /dev/null+++ b/pkg/writers/buffer_writer/metrics.go@@ -0,0 +1,24 @@+package bufferwriter++import (+ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+)++var (+ totalWriteSize = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "buffer_writer_total_write_size_bytes",+ Help: "Total size of data written by the BufferWriter in bytes.",+ })++ totalWriteDuration = promauto.NewCounter(prometheus.CounterOpts{+ Namespace: common.MetricsNamespace,+ Subsystem: common.MetricsSubsystem,+ Name: "buffer_writer_total_write_duration_microseconds",+ Help: "Total duration of write operations by the BufferWriter in microseconds.",+ })+)diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter.go b/pkg/writers/buffered_file_writer/bufferedfilewriter.goindex 088cc3ff8..d1c929340 100644--- a/pkg/writers/buffered_file_writer/bufferedfilewriter.go+++ b/pkg/writers/buffered_file_writer/bufferedfilewriter.go@@ -7,115 +7,18 @@ import ("fmt""io""os"- "sync""time""github.com/trufflesecurity/trufflehog/v3/pkg/cleantemp""github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffer")-type bufferPoolMetrics struct{}--func (bufferPoolMetrics) recordGrowth(growthAmount int) {- growCount.Inc()- growAmount.Add(float64(growthAmount))-}--func (bufferPoolMetrics) recordShrink(amount int) {- shrinkCount.Inc()- shrinkAmount.Add(float64(amount))-}--func (bufferPoolMetrics) recordCheckoutDuration(duration time.Duration) {- checkoutDuration.Observe(float64(duration.Microseconds()))- checkoutCount.Inc()- checkoutDurationTotal.Add(float64(duration.Microseconds()))-}--func (bufferPoolMetrics) recordBufferRetrival() {- activeBufferCount.Inc()- bufferCount.Inc()-}--func (bufferPoolMetrics) recordBufferReturn(bufCap, bufLen int64) {- activeBufferCount.Dec()- totalBufferSize.Add(float64(bufCap))- totalBufferLength.Add(float64(bufLen))-}--type bufPoolOpt func(pool *bufferPool)--type bufferPool struct {- bufferSize uint32- *sync.Pool-- metrics bufferPoolMetrics-}--const defaultBufferSize = 1 << 12 // 4KB-func newBufferPool(opts ...bufPoolOpt) *bufferPool {- pool := &bufferPool{bufferSize: defaultBufferSize}-- for _, opt := range opts {- opt(pool)- }- pool.Pool = &sync.Pool{- New: func() any {- buf := &buffer{Buffer: bytes.NewBuffer(make([]byte, 0, pool.bufferSize))}- return buf- },- }-- return pool-}-// sharedBufferPool is the shared buffer pool used by all BufferedFileWriters.// This allows for efficient reuse of buffers across multiple writers.-var sharedBufferPool *bufferPool--func init() { sharedBufferPool = newBufferPool() }--// buffer is a wrapper around bytes.Buffer that includes a timestamp for tracking buffer checkout duration.-type buffer struct {- *bytes.Buffer- checkedOut time.Time-}+var sharedBufferPool *buffer.Pool-func (bp *bufferPool) get(ctx context.Context) *buffer {- buf, ok := bp.Pool.Get().(*buffer)- if !ok {- ctx.Logger().Error(fmt.Errorf("buffer pool returned unexpected type"), "using new buffer")- buf = &buffer{Buffer: bytes.NewBuffer(make([]byte, 0, bp.bufferSize))}- }- buf.checkedOut = time.Now()- bp.metrics.recordBufferRetrival()-- return buf-}--func (bp *bufferPool) growBufferWithSize(buf *buffer, size int) {- // Grow the buffer to accommodate the new data.- bp.metrics.recordGrowth(size)- buf.Grow(size)-}--func (bp *bufferPool) put(buf *buffer) {- bp.metrics.recordBufferReturn(int64(buf.Cap()), int64(buf.Len()))- bp.metrics.recordCheckoutDuration(time.Since(buf.checkedOut))-- // If the buffer is more than twice the default size, replace it with a new buffer.- // This prevents us from returning very large buffers to the pool.- const maxAllowedCapacity = 2 * defaultBufferSize- if buf.Cap() > maxAllowedCapacity {- bp.metrics.recordShrink(buf.Cap() - defaultBufferSize)- buf = &buffer{Buffer: bytes.NewBuffer(make([]byte, 0, bp.bufferSize))}- } else {- // Reset the buffer to clear any existing data.- buf.Reset()- }-- bp.Put(buf)-}+func init() { sharedBufferPool = buffer.NewBufferPool() }type bufferedFileWriterMetrics struct{}@@ -150,8 +53,8 @@ type BufferedFileWriter struct {threshold uint64 // Threshold for switching to file writing.size uint64 // Total size of the data written.- bufPool *bufferPool // Pool for storing buffers for reuse.- buf *buffer // Buffer for storing data under the threshold in memory.+ bufPool *buffer.Pool // Pool for storing buffers for reuse.+ buf *buffer.Buffer // Buffer for storing data under the threshold in memory.filename string // Name of the temporary file.file io.WriteCloser // File for storing data over the threshold.@@ -170,10 +73,15 @@ func WithThreshold(threshold uint64) Option {const defaultThreshold = 10 * 1024 * 1024 // 10MB// New creates a new BufferedFileWriter with the given options.-func New(opts ...Option) *BufferedFileWriter {+func New(ctx context.Context, opts ...Option) *BufferedFileWriter {+ buf := sharedBufferPool.Get(ctx)+ if buf == nil {+ buf = buffer.NewBuffer()+ }w := &BufferedFileWriter{threshold: defaultThreshold,state: writeOnly,+ buf: buf,bufPool: sharedBufferPool,}for _, opt := range opts {@@ -220,13 +128,7 @@ func (w *BufferedFileWriter) Write(ctx context.Context, data []byte) (int, error}size := uint64(len(data))-- if w.buf == nil || w.buf.Len() == 0 {- w.buf = w.bufPool.get(ctx)- }-bufferLength := w.buf.Len()-start := time.Now()defer func(start time.Time) {w.metrics.recordDataProcessed(size, time.Since(start))@@ -242,32 +144,7 @@ func (w *BufferedFileWriter) Write(ctx context.Context, data []byte) (int, errortotalSizeNeeded := uint64(bufferLength) + sizeif totalSizeNeeded <= w.threshold {- // If the total size is within the threshold, write to the buffer.- ctx.Logger().V(4).Info(- "writing to buffer",- "data_size", size,- "content_size", bufferLength,- )-- availableSpace := w.buf.Cap() - bufferLength- growSize := int(totalSizeNeeded) - bufferLength- if growSize > availableSpace {- ctx.Logger().V(4).Info(- "buffer size exceeded, growing buffer",- "current_size", bufferLength,- "new_size", totalSizeNeeded,- "available_space", availableSpace,- "grow_size", growSize,- )- // We are manually growing the buffer so we can track the growth via metrics.- // Knowing the exact data size, we directly resize to fit it, rather than exponential growth- // which may require multiple allocations and copies if the size required is much larger- // than double the capacity. Our approach aligns with default behavior when growth sizes- // happen to match current capacity, retaining asymptotic efficiency benefits.- w.bufPool.growBufferWithSize(w.buf, growSize)- }-- return w.buf.Write(data)+ return w.buf.Write(ctx, data)}// Switch to file writing if threshold is exceeded.@@ -289,7 +166,7 @@ func (w *BufferedFileWriter) Write(ctx context.Context, data []byte) (int, errorif _, err := w.buf.WriteTo(w.file); err != nil {return 0, err}- w.bufPool.put(w.buf)+ w.bufPool.Put(w.buf)}}ctx.Logger().V(4).Info("writing to file", "data_size", size)@@ -336,10 +213,7 @@ func (w *BufferedFileWriter) ReadCloser() (io.ReadCloser, error) {}// Data is in memory.- return &bufferReadCloser{- Reader: bytes.NewReader(w.buf.Bytes()),- onClose: func() { w.bufPool.put(w.buf) },- }, nil+ return buffer.ReadCloser(w.buf.Bytes(), func() { w.bufPool.Put(w.buf) }), nil}// autoDeletingFileReader wraps an *os.File and deletes the file on Close.@@ -355,23 +229,3 @@ func (r *autoDeletingFileReader) Close() error {defer os.Remove(r.Name()) // Delete the file after closingreturn r.File.Close()}--// bufferReadCloser is a custom implementation of io.ReadCloser. It wraps a bytes.Reader-// for reading data from an in-memory buffer and includes an onClose callback.-// The onClose callback is used to return the buffer to the pool, ensuring buffer re-usability.-type bufferReadCloser struct {- *bytes.Reader- onClose func()-}--// Close implements the io.Closer interface. It calls the onClose callback to return the buffer-// to the pool, enabling buffer reuse. This method should be called by the consumers of ReadCloser-// once they have finished reading the data to ensure proper resource management.-func (brc *bufferReadCloser) Close() error {- if brc.onClose == nil {- return nil- }-- brc.onClose() // Return the buffer to the pool- return nil-}diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go b/pkg/writers/buffered_file_writer/bufferedfilewriter_test.goindex 1dcc07e69..9d6da38b8 100644--- a/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go+++ b/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go@@ -32,7 +32,7 @@ func TestBufferedFileWriterNewThreshold(t *testing.T) {tc := tct.Run(tc.name, func(t *testing.T) {t.Parallel()- writer := New(tc.options...)+ writer := New(context.Background(), tc.options...)assert.Equal(t, tc.expectedThreshold, writer.threshold)// The state should always be writeOnly when created.assert.Equal(t, writeOnly, writer.state)@@ -77,7 +77,7 @@ func TestBufferedFileWriterString(t *testing.T) {t.Run(tc.name, func(t *testing.T) {t.Parallel()ctx := context.Background()- writer := New(WithThreshold(tc.threshold))+ writer := New(ctx, WithThreshold(tc.threshold))// First write, should go to file if it exceeds the threshold._, err := writer.Write(ctx, tc.input)assert.NoError(t, err)@@ -109,7 +109,7 @@ func BenchmarkBufferedFileWriterString_BufferOnly_Small(b *testing.B) {data := bytes.Repeat([]byte("a"), smallBuffer)ctx := context.Background()- writer := New()+ writer := New(ctx)_, err := writer.Write(ctx, data)assert.NoError(b, err)@@ -127,7 +127,7 @@ func BenchmarkBufferedFileWriterString_BufferOnly_Small(b *testing.B) {func BenchmarkBufferedFileWriterString_BufferOnly_Medium(b *testing.B) {data := bytes.Repeat([]byte("a"), mediumBuffer)ctx := context.Background()- writer := New()+ writer := New(ctx)_, err := writer.Write(ctx, data)assert.NoError(b, err)@@ -146,7 +146,7 @@ func BenchmarkBufferedFileWriterString_OnlyFile_Small(b *testing.B) {data := bytes.Repeat([]byte("a"), smallFile)ctx := context.Background()- writer := New()+ writer := New(ctx)_, err := writer.Write(ctx, data)assert.NoError(b, err)@@ -165,7 +165,7 @@ func BenchmarkBufferedFileWriterString_OnlyFile_Medium(b *testing.B) {data := bytes.Repeat([]byte("a"), mediumFile)ctx := context.Background()- writer := New()+ writer := New(ctx)_, err := writer.Write(ctx, data)assert.NoError(b, err)@@ -184,7 +184,7 @@ func BenchmarkBufferedFileWriterString_BufferWithFile_Small(b *testing.B) {data := bytes.Repeat([]byte("a"), smallFile)ctx := context.Background()- writer := New()+ writer := New(ctx)_, err := writer.Write(ctx, data)assert.NoError(b, err)@@ -207,7 +207,7 @@ func BenchmarkBufferedFileWriterString_BufferWithFile_Medium(b *testing.B) {data := bytes.Repeat([]byte("a"), mediumFile)ctx := context.Background()- writer := New()+ writer := New(ctx)_, err := writer.Write(ctx, data)assert.NoError(b, err)@@ -252,7 +252,7 @@ func TestBufferedFileWriterLen(t *testing.T) {tc := tct.Run(tc.name, func(t *testing.T) {t.Parallel()- writer := New()+ writer := New(context.Background())_, err := writer.Write(context.Background(), tc.input)assert.NoError(t, err)@@ -270,7 +270,7 @@ func TestBufferedFileWriterWriteWithinThreshold(t *testing.T) {ctx := context.Background()data := []byte("hello world")- writer := New(WithThreshold(64))+ writer := New(ctx, WithThreshold(64))_, err := writer.Write(ctx, data)assert.NoError(t, err)@@ -285,7 +285,7 @@ func TestBufferedFileWriterWriteExceedsThreshold(t *testing.T) {ctx := context.Background()data := []byte("hello world")- writer := New(WithThreshold(5))+ writer := New(ctx, WithThreshold(5))_, err := writer.Write(ctx, data)assert.NoError(t, err)@@ -311,7 +311,7 @@ func TestBufferedFileWriterWriteAfterFlush(t *testing.T) {subsequentData := []byte("subsequent data")// Initialize writer with a threshold that initialData will exceed.- writer := New(WithThreshold(uint64(len(initialData) - 1)))+ writer := New(ctx, WithThreshold(uint64(len(initialData)-1)))_, err := writer.Write(ctx, initialData)assert.NoError(t, err)@@ -398,7 +398,7 @@ func TestBufferedFileWriterClose(t *testing.T) {tc := tct.Run(tc.name, func(t *testing.T) {t.Parallel()- writer := New(WithThreshold(threshold))+ writer := New(ctx, WithThreshold(threshold))tc.prepareWriter(writer)@@ -420,7 +420,7 @@ func TestBufferedFileWriterClose(t *testing.T) {func TestBufferedFileWriterStateTransitionOnClose(t *testing.T) {t.Parallel()- writer := New()+ writer := New(context.Background())// Initially, the writer should be in write-only mode.assert.Equal(t, writeOnly, writer.state)@@ -439,7 +439,7 @@ func TestBufferedFileWriterStateTransitionOnClose(t *testing.T) {func TestBufferedFileWriterWriteInReadOnlyState(t *testing.T) {t.Parallel()- writer := New()+ writer := New(context.Background())_ = writer.CloseForWriting() // Transition to read-only mode// Attempt to write in read-only mode.@@ -458,7 +458,7 @@ func BenchmarkBufferedFileWriterWriteLarge(b *testing.B) {for i := 0; i < b.N; i++ {// Threshold is smaller than the data size, data should get flushed to the file.- writer := New(WithThreshold(1024))+ writer := New(ctx, WithThreshold(1024))b.StartTimer(){@@ -488,7 +488,7 @@ func BenchmarkBufferedFileWriterWriteSmall(b *testing.B) {for i := 0; i < b.N; i++ {// Threshold is the same as the buffer size, data should always be written to the buffer.- writer := New(WithThreshold(1024 * 1024))+ writer := New(ctx, WithThreshold(1024*1024))b.StartTimer(){diff --git a/pkg/writers/buffered_file_writer/metrics.go b/pkg/writers/buffered_file_writer/metrics.goindex a940090e5..89954c4f2 100644--- a/pkg/writers/buffered_file_writer/metrics.go+++ b/pkg/writers/buffered_file_writer/metrics.go@@ -8,95 +8,17 @@ import ()var (- activeBufferCount = promauto.NewGauge(prometheus.GaugeOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "active_buffer_count",- Help: "Current number of active buffers.",- })-- bufferCount = promauto.NewGauge(prometheus.GaugeOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "buffer_count",- Help: "Total number of buffers managed by the pool.",- })-- totalBufferLength = promauto.NewGauge(prometheus.GaugeOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "total_buffer_length",- Help: "Total length of all buffers combined.",- })-- totalBufferSize = promauto.NewGauge(prometheus.GaugeOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "total_buffer_size",- Help: "Total size of all buffers combined.",- })-- growCount = promauto.NewCounter(prometheus.CounterOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "grow_count",- Help: "Total number of times buffers in the pool have grown.",- })-- growAmount = promauto.NewCounter(prometheus.CounterOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "grow_amount",- Help: "Total amount of bytes buffers in the pool have grown by.",- })-- shrinkCount = promauto.NewCounter(prometheus.CounterOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "shrink_count",- Help: "Total number of times buffers in the pool have shrunk.",- })-- shrinkAmount = promauto.NewCounter(prometheus.CounterOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "shrink_amount",- Help: "Total amount of bytes buffers in the pool have shrunk by.",- })-- checkoutDurationTotal = promauto.NewCounter(prometheus.CounterOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "checkout_duration_total_us",- Help: "Total duration in microseconds of buffer checkouts.",- })-- checkoutDuration = promauto.NewHistogram(prometheus.HistogramOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "checkout_duration_us",- Help: "Duration in microseconds of buffer checkouts.",- Buckets: []float64{50, 500, 5000},- })-- checkoutCount = promauto.NewCounter(prometheus.CounterOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "checkout_count",- Help: "Total number of buffer checkouts.",- })-totalWriteSize = promauto.NewCounter(prometheus.CounterOpts{Namespace: common.MetricsNamespace,Subsystem: common.MetricsSubsystem,- Name: "total_write_size_bytes",+ Name: "buffered_file_writer_total_write_size_bytes",Help: "Total size of data written by the BufferedFileWriter in bytes.",})totalWriteDuration = promauto.NewCounter(prometheus.CounterOpts{Namespace: common.MetricsNamespace,Subsystem: common.MetricsSubsystem,- Name: "total_write_duration_microseconds",+ Name: "buffered_file_writer_total_write_duration_microseconds",Help: "Total duration of write operations by the BufferedFileWriter in microseconds.",})