Skip to content

Find, verify, and analyze leaked credentials

20.4 KB
Raw Blame
package docker

import (
	"archive/tar"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/google/go-containerregistry/pkg/authn"
	"github.com/google/go-containerregistry/pkg/name"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/daemon"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	gzip "github.com/klauspost/pgzip"
	"golang.org/x/sync/errgroup"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/anypb"

	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
	"github.com/trufflesecurity/trufflehog/v3/pkg/common/glob"
	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)

const SourceType = sourcespb.SourceType_SOURCE_TYPE_DOCKER

type Source struct {
	name        string
	sourceId    sources.SourceID
	jobId       sources.JobID
	verify      bool
	concurrency int
	conn        sourcespb.Docker
	globFilter  *glob.Filter // Filter for excluding files based on glob patterns
	sources.Progress
	sources.CommonSourceUnitUnmarshaller
}

// Ensure the Source satisfies the interfaces at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)

// Type returns the type of source.
// It is used for matching source types in configuration and job input.
func (s *Source) Type() sourcespb.SourceType {
	return SourceType
}

func (s *Source) SourceID() sources.SourceID {
	return s.sourceId
}

func (s *Source) JobID() sources.JobID {
	return s.jobId
}

// Init initializes the source.
func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
	s.name = name
	s.sourceId = sourceId
	s.jobId = jobId
	s.verify = verify
	s.concurrency = concurrency

	jobIDStr := fmt.Sprint(s.jobId)

	// Reset metrics for this source at initialization time.
	dockerImagesScanned.WithLabelValues(s.name, jobIDStr).Set(0)
	dockerLayersScanned.WithLabelValues(s.name, jobIDStr).Set(0)
	dockerLayersEnumerated.WithLabelValues(s.name, jobIDStr).Set(0)
	dockerHistoryEntriesEnumerated.WithLabelValues(s.name, jobIDStr).Set(0)
	dockerImagesEnumerated.WithLabelValues(s.name, jobIDStr).Set(0)
	dockerHistoryEntriesScanned.WithLabelValues(s.name, jobIDStr).Set(0)

	if err := anypb.UnmarshalTo(connection, &s.conn, proto.UnmarshalOptions{}); err != nil {
		return fmt.Errorf("error unmarshalling connection: %w", err)
	}

	// Extract exclude paths from connection and compile glob patterns
	if paths := s.conn.GetExcludePaths(); len(paths) > 0 {
		var err error
		s.globFilter, err = glob.NewGlobFilter(glob.WithExcludeGlobs(paths...))
		if err != nil {
			return fmt.Errorf("error creating glob filter for exclude paths: %w", err)
		}
	}

	ctx.Logger().V(1).Info("Docker source initialized",
		"auth_method", authMethod(&s.conn),
		"image_count", len(s.conn.GetImages()),
		"namespace", s.conn.GetNamespace(),
		"exclude_paths", s.conn.GetExcludePaths(),
		"concurrency", s.concurrency,
	)

	return nil
}

// authMethod names the configured credential type without its values.
func authMethod(conn *sourcespb.Docker) string {
	switch conn.GetCredential().(type) {
	case *sourcespb.Docker_Unauthenticated:
		return "unauthenticated"
	case *sourcespb.Docker_BasicAuth:
		return "basic_auth"
	case *sourcespb.Docker_BearerToken:
		return "bearer_token"
	case *sourcespb.Docker_DockerKeychain:
		return "docker_keychain"
	default:
		return "none"
	}
}

// imageInfo holds information about a Docker image being processed
type imageInfo struct {
	image v1.Image // The image object from go-containerregistry
	base  string   // Base name of the image (without tag/digest)
	tag   string   // Tag or digest of the image
}

// historyEntryInfo represents a single entry from the image's build history
type historyEntryInfo struct {
	index       int        // Position in the history array
	entry       v1.History // The history entry containing build commands
	layerDigest string     // SHA256 digest of the associated layer (empty for empty layers)
	base        string     // Base name of the image
	tag         string     // Tag or digest of the image
}

// layerInfo contains identifying information for a Docker image layer
type layerInfo struct {
	digest v1.Hash // SHA256 digest uniquely identifying the layer
	base   string  // Base name of the image
	tag    string  // Tag or digest of the image
}

// Chunks emits data over a channel that is decoded and scanned for secrets.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
	jobIDStr := fmt.Sprint(s.jobId)

	workers := new(errgroup.Group)
	workers.SetLimit(s.concurrency)

	// if namespace is set and no images are specified, fetch all images in that namespace.
	registryNamespace := s.conn.GetNamespace()
	if registryNamespace != "" && len(s.conn.Images) == 0 {
		start := time.Now()
		namespaceImages, err := GetNamespaceImages(ctx, registryNamespace, s.conn.GetRegistryToken())
		if err != nil {
			return fmt.Errorf("failed to list namespace: %s images: %w", registryNamespace, err)
		}

		dockerListImagesAPIDuration.WithLabelValues(s.name).Observe(time.Since(start).Seconds())

		s.conn.Images = append(s.conn.Images, namespaceImages...)
	}

	for _, image := range s.conn.GetImages() {
		if common.IsDone(ctx) {
			return nil
		}

		imgInfo, err := s.processImage(ctx, image)
		if err != nil {
			ctx.Logger().Error(err, "error processing image", "image", image)
			continue
		}

		imageStart := time.Now()
		imageCtx := context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)

		layers, err := imgInfo.image.Layers()
		if err != nil {
			imageCtx.Logger().Error(err, "error getting image layers")
			continue
		}
		dockerLayersEnumerated.WithLabelValues(s.name, jobIDStr).Add(float64(len(layers)))

		// Get history entries and associate them with layers
		historyEntries, err := getHistoryEntries(imageCtx, imgInfo, layers)
		if err != nil {
			imageCtx.Logger().Error(err, "error getting image history entries")
			continue
		}
		dockerHistoryEntriesEnumerated.WithLabelValues(s.name, jobIDStr).Add(float64(len(historyEntries)))

		imageCtx.Logger().V(2).Info("Scanning image", "layers", len(layers), "history_entries", len(historyEntries))

		// Scan each history entry for secrets in build commands
		for _, historyEntry := range historyEntries {
			if err := s.processHistoryEntry(imageCtx, historyEntry, chunksChan); err != nil {
				imageCtx.Logger().Error(err, "error processing history entry")
				continue
			}
			dockerHistoryEntriesScanned.WithLabelValues(s.name, jobIDStr).Inc()
		}

		// Process each layer concurrently
		for _, layer := range layers {
			workers.Go(func() error {
				if err := s.processLayer(imageCtx, layer, imgInfo, chunksChan); err != nil {
					// a cancelled scan fails every layer left, which is not a layer error
					if imageCtx.Err() == nil {
						imageCtx.Logger().Error(err, "error processing layer")
					}
					return nil
				}
				dockerLayersScanned.WithLabelValues(s.name, jobIDStr).Inc()

				return nil
			})
		}

		if err := workers.Wait(); err != nil {
			imageCtx.Logger().Error(err, "error processing layers")
			continue
		}

		dockerImagesScanned.WithLabelValues(s.name, jobIDStr).Inc()
		imageCtx.Logger().V(2).Info("Finished scanning image", "duration", time.Since(imageStart))
	}

	return nil
}

// processImage processes an individual image and prepares it for further processing.
// It handles three image source types: remote registry, local daemon, and tarball file.
func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
	var (
		imgInfo   imageInfo
		imageName name.Reference
		err       error
	)

	remoteOpts, err := s.remoteOpts(ctx)
	if err != nil {
		return imgInfo, err
	}

	const filePrefix = "file://"
	const dockerPrefix = "docker://"

	// Handle tarball file images
	if image, ok := strings.CutPrefix(image, filePrefix); ok {
		ctx.Logger().V(5).Info("Loading image", "image", image, "image_source", "tarball")
		imgInfo.base = image
		imgInfo.image, err = tarball.ImageFromPath(image, nil)
		if err != nil {
			return imgInfo, err
		}
		// Handle local Docker daemon images
	} else if image, ok := strings.CutPrefix(image, dockerPrefix); ok {
		ctx.Logger().V(5).Info("Loading image", "image", image, "image_source", "daemon")
		imgInfo, imageName, err = s.extractImageNameTagDigest(image)
		if err != nil {
			return imgInfo, err
		}
		imgInfo.image, err = daemon.Image(imageName, daemon.WithContext(ctx))
		if err != nil {
			return imgInfo, err
		}
		// Handle remote registry images (default)
	} else {
		ctx.Logger().V(5).Info("Loading image", "image", image, "image_source", "remote")
		imgInfo, imageName, err = s.extractImageNameTagDigest(image)
		if err != nil {
			return imgInfo, err
		}
		imgInfo.image, err = remote.Image(imageName, remoteOpts...)
		if err != nil {
			return imgInfo, err
		}
	}

	return imgInfo, nil
}

// extractImageNameTagDigest parses the provided Docker image string and returns a name.Reference
// representing either the image's tag or digest, and any error encountered during parsing.
func (*Source) extractImageNameTagDigest(image string) (imageInfo, name.Reference, error) {
	var (
		hasDigest bool
		imgInfo   imageInfo
		imgName   name.Reference
		err       error
	)
	imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)

	// Parse as digest reference (e.g., nginx@sha256:abc123...)
	if hasDigest {
		imgName, err = name.NewDigest(image)
		// Parse as tag reference (e.g., nginx:latest)
	} else {
		imgName, err = name.NewTag(image)
	}
	if err != nil {
		return imgInfo, imgName, err
	}

	return imgInfo, imgName, nil
}

// getHistoryEntries collates an image's configuration history together with the
// corresponding layer digests for any non-empty layers.
func getHistoryEntries(ctx context.Context, imgInfo imageInfo, layers []v1.Layer) ([]historyEntryInfo, error) {
	ctx.Logger().V(5).Info("Getting history entries")
	config, err := imgInfo.image.ConfigFile()
	if err != nil {
		return nil, err
	}

	history := config.History
	entries := make([]historyEntryInfo, len(history))

	layerIndex := 0
	for historyIndex, entry := range history {
		e := historyEntryInfo{
			base:  imgInfo.base,
			tag:   imgInfo.tag,
			entry: entry,
			index: historyIndex,
		}

		// Associate with a layer if possible. Some history entries don't create layers (e.g., ENV, LABEL)
		// Failing to associate won't affect scanning, just reduces traceability
		if !entry.EmptyLayer {
			if layerIndex < len(layers) {
				digest, err := layers[layerIndex].Digest()

				if err == nil {
					e.layerDigest = digest.String()
				} else {
					ctx.Logger().Error(err, "cannot associate layer with history entry: layer digest failed",
						"layerIndex", layerIndex, "historyIndex", historyIndex)
				}
			} else {
				ctx.Logger().V(4).Info("cannot associate layer with history entry: no correlated layer exists at this index",
					"layerIndex", layerIndex, "historyIndex", historyIndex)
			}

			layerIndex++
		}

		entries[historyIndex] = e
	}

	return entries, nil
}

// processHistoryEntry processes a history entry from the image configuration metadata.
// It scans the CreatedBy field which contains the command used to create that layer.
func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEntryInfo, chunksChan chan *sources.Chunk) error {
	// Create a descriptive identifier for this history entry
	// There's no file name here, so we use a synthetic path
	entryPath := fmt.Sprintf("image-metadata:history:%d:created-by", historyInfo.index)

	chunk := &sources.Chunk{
		SourceType: s.Type(),
		SourceName: s.name,
		SourceID:   s.SourceID(),
		SourceMetadata: &source_metadatapb.MetaData{
			Data: &source_metadatapb.MetaData_Docker{
				Docker: &source_metadatapb.Docker{
					File:  entryPath,
					Image: historyInfo.base,
					Tag:   historyInfo.tag,
					Layer: historyInfo.layerDigest,
				},
			},
		},
		SourceVerify: s.verify,
		Data:         []byte(historyInfo.entry.CreatedBy),
	}

	ctx.Logger().V(3).Info("Scanning image history entry", "history_index", historyInfo.index, "layer", historyInfo.layerDigest)

	return common.CancellableWrite(ctx, chunksChan, chunk)
}

// processLayer processes an individual layer of an image.
// It decompresses the layer and extracts all files for scanning.
func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
	start := time.Now()
	layerInfo := layerInfo{
		base: imgInfo.base,
		tag:  imgInfo.tag,
	}

	var err error
	layerInfo.digest, err = layer.Digest()
	if err != nil {
		return err
	}

	ctx = context.WithValues(ctx, "layer", layerInfo.digest.String())
	ctx.Logger().V(3).Info("Scanning layer")

	rc, err := layer.Compressed()
	if err != nil {
		return err
	}
	defer func() { _ = rc.Close() }()

	// Configure parallel gzip decompression for better performance
	const (
		defaultBlockSize = 1 << 24 // 16MB blocks
		defaultBlocks    = 8       // Process 8 blocks in parallel
	)

	gzipReader, err := gzip.NewReaderN(rc, defaultBlockSize, defaultBlocks)
	if err != nil {
		return err
	}
	defer func() { _ = gzipReader.Close() }()

	// Layers are tar archives, so we read them as tar files
	tarReader := tar.NewReader(gzipReader)
	var files, skipped int
	for {
		header, err := tarReader.Next()
		if errors.Is(err, io.EOF) {
			break
		}
		if err != nil {
			return err
		}

		info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
		scanned, err := s.processChunk(ctx, info, chunksChan)
		if err != nil {
			return err
		}
		files++
		if !scanned {
			skipped++
		}
	}

	ctx.Logger().V(3).Info("Finished scanning layer", "files", files, "skipped", skipped, "duration", time.Since(start))
	return nil
}

// chunkProcessingInfo holds information needed to process a single file from a layer
type chunkProcessingInfo struct {
	size   int64     // Size of the file in bytes
	name   string    // Full path of the file within the layer
	reader io.Reader // Reader for the file contents
	layer  layerInfo // Information about the layer this file belongs to
}

// processChunk processes an individual chunk of a layer.
// It applies file size limits and exclusion patterns before scanning, and
// reports whether the file was scanned.
func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) (bool, error) {
	// Skip files that are too large to avoid memory issues
	const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
	if info.size > filesizeLimitBytes {
		ctx.Logger().V(4).Info("Skipping file", "file", info.name, "reason", "too large", "size", info.size, "limit", filesizeLimitBytes)
		return false, nil
	}

	// Check if the file matches any exclude patterns
	filePath := "/" + info.name
	if s.isExcluded(ctx, filePath) {
		return false, nil
	}

	ctx.Logger().V(5).Info("Scanning file", "file", filePath, "size", info.size)

	// Read the file in chunks to handle large files efficiently
	chunkReader := sources.NewChunkReader(sources.WithFileSize(int(info.size)))
	chunkResChan := chunkReader(ctx, info.reader)

	for data := range chunkResChan {
		if err := data.Error(); err != nil {
			ctx.Logger().Error(err, "error reading chunk", "file", filePath)
			continue
		}

		chunk := &sources.Chunk{
			SourceType: s.Type(),
			SourceName: s.name,
			SourceID:   s.SourceID(),
			SourceMetadata: &source_metadatapb.MetaData{
				Data: &source_metadatapb.MetaData_Docker{
					Docker: &source_metadatapb.Docker{
						File:  "/" + info.name,
						Image: info.layer.base,
						Tag:   info.layer.tag,
						Layer: info.layer.digest.String(),
					},
				},
			},
			SourceVerify: s.verify,
		}
		chunk.Data = data.Bytes()

		if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
			return true, err
		}
	}

	return true, nil
}

// isExcluded checks if a given filePath should be excluded based on the configured glob patterns.
func (s *Source) isExcluded(ctx context.Context, filePath string) bool {
	if s.globFilter == nil {
		return false // No filter configured, so nothing is excluded
	}

	// ShouldInclude returns true if the file should be scanned (not excluded)
	// If it returns false, the file matches an exclude pattern
	isIncluded := s.globFilter.ShouldInclude(filePath)

	if !isIncluded {
		ctx.Logger().V(4).Info("Skipping file", "file", filePath, "reason", "matches exclude pattern")
	}
	return !isIncluded
}

// remoteOpts configures the options for fetching images from remote registries.
// It sets up HTTP transport and authentication based on the connection configuration.
func (s *Source) remoteOpts(ctx context.Context) ([]remote.Option, error) {
	// Configure HTTP transport with reasonable timeouts and connection pooling
	defaultTransport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			Timeout:   30 * time.Second,
			KeepAlive: 30 * time.Second,
		}).DialContext,
		ForceAttemptHTTP2:     true,
		MaxIdleConns:          s.concurrency * 4,
		IdleConnTimeout:       90 * time.Second,
		TLSHandshakeTimeout:   10 * time.Second,
		ExpectContinueTimeout: 1 * time.Second,
		MaxIdleConnsPerHost:   s.concurrency * 2,
	}

	// The context lets a cancelled scan stop a slow pull, and lets the
	// registry log lines carry the source fields.
	opts := []remote.Option{
		remote.WithContext(ctx),
		remote.WithTransport(&registryLogTransport{
			T: common.NewInstrumentedTransport(common.NewCustomTransport(defaultTransport)),
		}),
	}

	// Configure authentication based on credential type
	switch s.conn.GetCredential().(type) {
	case *sourcespb.Docker_Unauthenticated:
		// no auth option, but keep the transport and context
	case *sourcespb.Docker_BasicAuth:
		opts = append(opts, remote.WithAuth(&authn.Basic{
			Username: s.conn.GetBasicAuth().GetUsername(),
			Password: s.conn.GetBasicAuth().GetPassword(),
		}))
	case *sourcespb.Docker_BearerToken:
		opts = append(opts, remote.WithAuth(&authn.Bearer{
			Token: s.conn.GetBearerToken(),
		}))
	case *sourcespb.Docker_DockerKeychain:
		// Use credentials from ~/.docker/config.json
		opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain))
	default:
		return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
	}

	return opts, nil
}

func GetNamespaceImages(ctx context.Context, namespace, registryToken string) ([]string, error) {
	start := time.Now()
	registry := MakeRegistryFromNamespace(namespace)

	// attach the registry authentication token, if one is available.
	if registryToken != "" {
		registry.WithRegistryToken(registryToken)
	}

	ctx.Logger().V(1).Info("Listing namespace images", "registry", registry.Name(), "namespace", namespace)

	namespaceImages, err := registry.ListImages(ctx, namespace)
	if err != nil {
		return nil, fmt.Errorf("failed to list namespace images: %w", err)
	}

	ctx.Logger().V(1).Info("Listed namespace images",
		"registry", registry.Name(),
		"namespace", namespace,
		"count", len(namespaceImages),
		"duration", time.Since(start),
	)

	return namespaceImages, nil
}

// baseAndTagFromImage extracts the base name and tag/digest from an image reference string.
// It handles both digest-based references (image@sha256:...) and tag-based references (image:tag).
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
	if base, tag, hasDigest = extractDigest(image); hasDigest {
		return base, tag, true
	}

	base, tag = extractTagOrUseDefault(image)
	return base, tag, false
}

// extractDigest tries to split the image string on the digest delimiter (@).
// If successful, it means the image has a digest reference.
func extractDigest(image string) (base, tag string, hasDigest bool) {
	const digestDelim = "@"

	if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
		return parts[0], parts[1], true
	}
	return "", "", false
}

// extractTagOrUseDefault extracts the tag from the image string.
// If no tag is found, it defaults to "latest".
func extractTagOrUseDefault(image string) (base, tag string) {
	const (
		tagDelim     = ":"
		regRepoDelim = "/"
	)

	parts := strings.Split(image, tagDelim)

	// Check if the last part is a tag (not a hostname with port)
	// We use a weak validation: if it contains "/" it's likely part of a registry hostname
	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
		return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
	}
	return image, "latest"
}