Skip to content

Modern Docker Management, Designed for Everyone

fix: use temp file for trivy output (#1751)

Kyle Mendella43551e56c25
diff --git a/backend/internal/services/vulnerability_service.go b/backend/internal/services/vulnerability_service.go
index d88bdb7e6..cd9c96bf2 100644
--- a/backend/internal/services/vulnerability_service.go
+++ b/backend/internal/services/vulnerability_service.go
@@ -1,13 +1,13 @@
package services
import (
- "bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
+ "net"
"os"
"strings"
"sync"
@@ -36,6 +36,7 @@ const (
scanStaleTimeout = 30 * time.Minute
trivyMaxCPUNano = int64(1_000_000_000) // 1 CPU core
trivyMaxMemoryBytes = int64(512 * 1024 * 1024)
+ trivyErrorExcerptSize = int64(32 * 1024)
)
// VulnerabilityService handles vulnerability scanning of container images
@@ -914,6 +915,18 @@ func (s *VulnerabilityService) execTrivyScanInContainer(ctx context.Context, con
return nil, fmt.Errorf("failed to connect to Docker: %w", err)
}
+ stdoutFile, err := createTrivyLogTempFileInternal("trivy-exec-stdout-*")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create trivy exec stdout temp file: %w", err)
+ }
+
+ stderrFile, err := createTrivyLogTempFileInternal("trivy-exec-stderr-*")
+ if err != nil {
+ cleanupTrivyLogTempFilesInternal(ctx, stdoutFile)
+ return nil, fmt.Errorf("failed to create trivy exec stderr temp file: %w", err)
+ }
+ defer cleanupTrivyLogTempFilesInternal(ctx, stdoutFile, stderrFile)
+
trivyTimeout := s.getTrivyScanTimeoutArg()
execCfg := containertypes.ExecOptions{
@@ -933,8 +946,8 @@ func (s *VulnerabilityService) execTrivyScanInContainer(ctx context.Context, con
}
defer attachResp.Close()
- var stdout, stderr bytes.Buffer
- if _, err := stdcopy.StdCopy(&stdout, &stderr, attachResp.Reader); err != nil {
+ logDone := startDockerLogCopyInternal(attachResp.Reader, stdoutFile, stderrFile)
+ if err := waitForDockerLogCopyInternal(logDone); err != nil {
return nil, fmt.Errorf("failed to read exec output: %w", err)
}
@@ -945,34 +958,45 @@ func (s *VulnerabilityService) execTrivyScanInContainer(ctx context.Context, con
}
if execInspect.ExitCode != 0 {
- errMsg := strings.TrimSpace(stderr.String())
+ errMsg := readTempFileExcerptInternal(stderrFile, trivyErrorExcerptSize)
if errMsg == "" {
- errMsg = strings.TrimSpace(stdout.String())
+ errMsg = readTempFileExcerptInternal(stdoutFile, trivyErrorExcerptSize)
}
if errMsg == "" {
errMsg = fmt.Sprintf("exit status %d", execInspect.ExitCode)
}
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, int64(execInspect.ExitCode), stdoutFile, stderrFile, "exec non-zero exit")
if strings.Contains(strings.ToLower(errMsg), "deadline exceeded") {
return nil, fmt.Errorf("trivy scan timed out for %s (increase TRIVY_SCAN_TIMEOUT or trivyScanTimeout setting)", imageName)
}
return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
}
- output := bytes.TrimSpace(stdout.Bytes())
- if len(output) == 0 {
- errMsg := strings.TrimSpace(stderr.String())
+ trivyReport, err := decodeTrivyReportFromFileInternal(stdoutFile)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ errMsg := readTempFileExcerptInternal(stderrFile, trivyErrorExcerptSize)
+ if errMsg == "" {
+ errMsg = "trivy scan produced no output"
+ }
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, int64(execInspect.ExitCode), stdoutFile, stderrFile, "exec empty output")
+ return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
+ }
+
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, int64(execInspect.ExitCode), stdoutFile, stderrFile, "exec parse error")
+ return nil, fmt.Errorf("failed to parse trivy output: %w", err)
+ }
+
+ if trivyReport == nil {
+ errMsg := readTempFileExcerptInternal(stderrFile, trivyErrorExcerptSize)
if errMsg == "" {
errMsg = "trivy scan produced no output"
}
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, int64(execInspect.ExitCode), stdoutFile, stderrFile, "exec nil report")
return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
}
- var trivyReport vulnerability.TrivyReport
- if err := json.Unmarshal(output, &trivyReport); err != nil {
- return nil, fmt.Errorf("failed to parse trivy output: %w", err)
- }
-
- result := vulnerability.ConvertTrivyReportToScanResult(&trivyReport, imageID, time.Now(), 0)
+ result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), 0)
result.ScannerVersion = scannerVersion
return result, nil
@@ -1078,12 +1102,19 @@ func (s *VulnerabilityService) GetTrivyVersion(ctx context.Context) string {
}
defer logs.Close()
- var stdout bytes.Buffer
- logDone := make(chan error, 1)
- go func() {
- _, err := stdcopy.StdCopy(&stdout, io.Discard, logs)
- logDone <- err
- }()
+ stdoutFile, err := createTrivyLogTempFileInternal("trivy-version-stdout-*")
+ if err != nil {
+ return ""
+ }
+
+ stderrFile, err := createTrivyLogTempFileInternal("trivy-version-stderr-*")
+ if err != nil {
+ cleanupTrivyLogTempFilesInternal(ctx, stdoutFile)
+ return ""
+ }
+ defer cleanupTrivyLogTempFilesInternal(ctx, stdoutFile, stderrFile)
+
+ logDone := startDockerLogCopyInternal(logs, stdoutFile, stderrFile)
statusCh, errCh := dockerClient.ContainerWait(ctx, resp.ID, containertypes.WaitConditionNotRunning)
var waitResp containertypes.WaitResponse
@@ -1095,8 +1126,7 @@ func (s *VulnerabilityService) GetTrivyVersion(ctx context.Context) string {
case waitResp = <-statusCh:
}
- logs.Close()
- if err := <-logDone; err != nil && !errors.Is(err, io.EOF) {
+ if err := waitForDockerLogCopyInternal(logDone); err != nil {
return ""
}
@@ -1104,7 +1134,7 @@ func (s *VulnerabilityService) GetTrivyVersion(ctx context.Context) string {
return ""
}
- return parseTrivyVersion(stdout.String())
+ return parseTrivyVersion(readTempFileExcerptInternal(stdoutFile, trivyErrorExcerptSize))
}
func parseTrivyVersion(output string) string {
@@ -1342,12 +1372,164 @@ func addTrivyTempFileMounts(hostConfig *containertypes.HostConfig, tempFiles []s
}
}
+func createTrivyLogTempFileInternal(prefix string) (*os.File, error) {
+ file, err := os.CreateTemp("", prefix)
+ if err != nil {
+ return nil, err
+ }
+ return file, nil
+}
+
+func cleanupTrivyLogTempFilesInternal(ctx context.Context, files ...*os.File) {
+ for _, file := range files {
+ if file == nil {
+ continue
+ }
+
+ path := file.Name()
+ if err := file.Close(); err != nil {
+ slog.WarnContext(ctx, "failed to close trivy temp file", "path", path, "error", err)
+ }
+
+ if path == "" {
+ continue
+ }
+
+ if err := os.Remove(path); err != nil {
+ slog.WarnContext(ctx, "failed to remove trivy temp file", "path", path, "error", err)
+ }
+ }
+}
+
+func startDockerLogCopyInternal(stream io.Reader, stdoutWriter, stderrWriter io.Writer) <-chan error {
+ logDone := make(chan error, 1)
+ go func() {
+ _, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, stream)
+ logDone <- err
+ }()
+ return logDone
+}
+
+func waitForDockerLogCopyInternal(logDone <-chan error) error {
+ err := <-logDone
+ if err == nil || isExpectedDockerStreamEndErrorInternal(err) {
+ return nil
+ }
+ return err
+}
+
+func isExpectedDockerStreamEndErrorInternal(err error) bool {
+ if err == nil {
+ return false
+ }
+
+ if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
+ return true
+ }
+
+ errMsg := strings.ToLower(err.Error())
+ return strings.Contains(errMsg, "use of closed network connection") ||
+ strings.Contains(errMsg, "broken pipe") ||
+ strings.Contains(errMsg, "connection reset by peer")
+}
+
+func tempFileSizeInternal(file *os.File) int64 {
+ if file == nil {
+ return 0
+ }
+
+ stat, err := file.Stat()
+ if err != nil {
+ return 0
+ }
+
+ return stat.Size()
+}
+
+func readTempFileExcerptInternal(file *os.File, maxBytes int64) string {
+ if file == nil {
+ return ""
+ }
+
+ if _, err := file.Seek(0, io.SeekStart); err != nil {
+ return ""
+ }
+
+ reader := io.Reader(file)
+ if maxBytes > 0 {
+ reader = io.LimitReader(file, maxBytes)
+ }
+
+ content, err := io.ReadAll(reader)
+ if err != nil {
+ return ""
+ }
+
+ return strings.TrimSpace(string(content))
+}
+
+func decodeTrivyReportFromFileInternal(file *os.File) (*vulnerability.TrivyReport, error) {
+ if file == nil {
+ return nil, errors.New("trivy output file is nil")
+ }
+
+ if _, err := file.Seek(0, io.SeekStart); err != nil {
+ return nil, fmt.Errorf("failed to seek trivy output file: %w", err)
+ }
+
+ decoder := json.NewDecoder(file)
+ var report vulnerability.TrivyReport
+ if err := decoder.Decode(&report); err != nil {
+ return nil, err
+ }
+
+ var trailing json.RawMessage
+ if err := decoder.Decode(&trailing); err != nil && !errors.Is(err, io.EOF) {
+ return nil, fmt.Errorf("trivy output contains trailing data: %w", err)
+ }
+ if len(trailing) > 0 {
+ return nil, errors.New("trivy output contains trailing data")
+ }
+
+ return &report, nil
+}
+
+func logTrivyScanFailureDiagnosticsInternal(ctx context.Context, imageName, imageID string, exitCode int64, stdoutFile, stderrFile *os.File, reason string) {
+ slog.WarnContext(ctx,
+ "trivy scan diagnostics",
+ "image", imageName,
+ "imageId", imageID,
+ "reason", reason,
+ "exitCode", exitCode,
+ "stdoutBytes", tempFileSizeInternal(stdoutFile),
+ "stderrBytes", tempFileSizeInternal(stderrFile),
+ )
+}
+
func (s *VulnerabilityService) runTrivyContainer(
ctx context.Context,
dockerClient *client.Client,
config *containertypes.Config,
hostConfig *containertypes.HostConfig,
-) ([]byte, []byte, int64, int64, error) {
+) (*os.File, *os.File, int64, int64, error) {
+ stdoutFile, err := createTrivyLogTempFileInternal("trivy-stdout-*")
+ if err != nil {
+ return nil, nil, 0, 0, fmt.Errorf("failed to create trivy stdout temp file: %w", err)
+ }
+
+ stderrFile, err := createTrivyLogTempFileInternal("trivy-stderr-*")
+ if err != nil {
+ cleanupTrivyLogTempFilesInternal(ctx, stdoutFile)
+ return nil, nil, 0, 0, fmt.Errorf("failed to create trivy stderr temp file: %w", err)
+ }
+
+ success := false
+ defer func() {
+ if !success {
+ cleanupTrivyLogTempFilesInternal(ctx, stdoutFile, stderrFile)
+ }
+ }()
+
resp, err := dockerClient.ContainerCreate(ctx, config, hostConfig, nil, nil, "")
if err != nil {
return nil, nil, 0, 0, fmt.Errorf("failed to create trivy container: %w", err)
@@ -1369,12 +1551,7 @@ func (s *VulnerabilityService) runTrivyContainer(
}
defer logs.Close()
- var stdout, stderr bytes.Buffer
- logDone := make(chan error, 1)
- go func() {
- _, err := stdcopy.StdCopy(&stdout, &stderr, logs)
- logDone <- err
- }()
+ logDone := startDockerLogCopyInternal(logs, stdoutFile, stderrFile)
startTime := time.Now()
statusCode, err := s.waitForTrivyContainer(ctx, dockerClient, resp.ID)
@@ -1382,13 +1559,13 @@ func (s *VulnerabilityService) runTrivyContainer(
return nil, nil, 0, 0, err
}
- logs.Close()
- if err := <-logDone; err != nil && !errors.Is(err, io.EOF) {
+ if err := waitForDockerLogCopyInternal(logDone); err != nil {
return nil, nil, 0, 0, fmt.Errorf("failed to read trivy logs: %w", err)
}
duration := time.Since(startTime).Milliseconds()
- return stdout.Bytes(), stderr.Bytes(), duration, statusCode, nil
+ success = true
+ return stdoutFile, stderrFile, duration, statusCode, nil
}
func (s *VulnerabilityService) waitForTrivyContainer(
@@ -1448,42 +1625,53 @@ func (s *VulnerabilityService) runTrivyScan(ctx context.Context, trivyImage stri
config := buildTrivyContainerConfig(trivyImage, cmdArgs)
hostConfig := buildTrivyHostConfig(cacheVolume, tempFiles)
- stdout, stderr, duration, statusCode, err := s.runTrivyContainer(ctx, dockerClient, config, hostConfig)
+ stdoutFile, stderrFile, duration, statusCode, err := s.runTrivyContainer(ctx, dockerClient, config, hostConfig)
if err != nil {
return nil, err
}
+ defer cleanupTrivyLogTempFilesInternal(ctx, stdoutFile, stderrFile)
if statusCode != 0 {
- errMsg := strings.TrimSpace(string(stderr))
+ errMsg := readTempFileExcerptInternal(stderrFile, trivyErrorExcerptSize)
if errMsg == "" {
- errMsg = strings.TrimSpace(string(stdout))
+ errMsg = readTempFileExcerptInternal(stdoutFile, trivyErrorExcerptSize)
}
if errMsg == "" {
errMsg = fmt.Sprintf("exit status %d", statusCode)
}
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, statusCode, stdoutFile, stderrFile, "container non-zero exit")
if strings.Contains(strings.ToLower(errMsg), "deadline exceeded") {
return nil, fmt.Errorf("trivy scan timed out for %s (increase TRIVY_SCAN_TIMEOUT or trivyScanTimeout setting)", imageName)
}
return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
}
- output := bytes.TrimSpace(stdout)
- if len(output) == 0 {
- errMsg := strings.TrimSpace(string(stderr))
+ trivyReport, err := decodeTrivyReportFromFileInternal(stdoutFile)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ errMsg := readTempFileExcerptInternal(stderrFile, trivyErrorExcerptSize)
+ if errMsg == "" {
+ errMsg = "trivy scan produced no output"
+ }
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, statusCode, stdoutFile, stderrFile, "container empty output")
+ return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
+ }
+
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, statusCode, stdoutFile, stderrFile, "container parse error")
+ return nil, fmt.Errorf("failed to parse trivy output: %w", err)
+ }
+
+ if trivyReport == nil {
+ errMsg := readTempFileExcerptInternal(stderrFile, trivyErrorExcerptSize)
if errMsg == "" {
errMsg = "trivy scan produced no output"
}
+ logTrivyScanFailureDiagnosticsInternal(ctx, imageName, imageID, statusCode, stdoutFile, stderrFile, "container nil report")
return nil, fmt.Errorf("trivy scan failed: %s", errMsg)
}
- // Parse Trivy JSON output
- var trivyReport vulnerability.TrivyReport
- if err := json.Unmarshal(output, &trivyReport); err != nil {
- return nil, fmt.Errorf("failed to parse trivy output: %w", err)
- }
-
// Convert Trivy report to our ScanResult format
- result := vulnerability.ConvertTrivyReportToScanResult(&trivyReport, imageID, time.Now(), duration)
+ result := vulnerability.ConvertTrivyReportToScanResult(trivyReport, imageID, time.Now(), duration)
result.ScannerVersion = s.GetTrivyVersion(ctx)
return result, nil
diff --git a/backend/internal/services/vulnerability_service_test.go b/backend/internal/services/vulnerability_service_test.go
new file mode 100644
index 000000000..e78d00462
--- /dev/null
+++ b/backend/internal/services/vulnerability_service_test.go
@@ -0,0 +1,110 @@
+package services
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "os"
+ "testing"
+
+ "github.com/getarcaneapp/arcane/types/vulnerability"
+ "github.com/stretchr/testify/require"
+)
+
+func TestIsExpectedDockerStreamEndErrorInternal(t *testing.T) {
+ tests := []struct {
+ name string
+ err error
+ want bool
+ }{
+ {name: "nil", err: nil, want: false},
+ {name: "eof", err: io.EOF, want: true},
+ {name: "wrapped eof", err: fmt.Errorf("wrapped: %w", io.EOF), want: true},
+ {name: "net err closed", err: net.ErrClosed, want: true},
+ {name: "closed network connection", err: errors.New("read tcp 127.0.0.1:1234->127.0.0.1:5678: use of closed network connection"), want: true},
+ {name: "broken pipe", err: errors.New("write: broken pipe"), want: true},
+ {name: "connection reset by peer", err: errors.New("read: connection reset by peer"), want: true},
+ {name: "unexpected", err: errors.New("some other error"), want: false},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ require.Equal(t, tt.want, isExpectedDockerStreamEndErrorInternal(tt.err))
+ })
+ }
+}
+
+func TestDecodeTrivyReportFromFileInternal_LargePayload(t *testing.T) {
+ tmpFile, err := os.CreateTemp("", "trivy-report-*.json")
+ require.NoError(t, err)
+ defer func() {
+ require.NoError(t, os.Remove(tmpFile.Name()))
+ }()
+ defer func() {
+ require.NoError(t, tmpFile.Close())
+ }()
+
+ const vulnCount = 4000
+ vulns := make([]vulnerability.TrivyVulnerability, 0, vulnCount)
+ for i := 0; i < vulnCount; i++ {
+ vulns = append(vulns, vulnerability.TrivyVulnerability{
+ VulnerabilityID: fmt.Sprintf("CVE-2026-%04d", i),
+ PkgName: fmt.Sprintf("pkg-%d", i),
+ InstalledVersion: "1.0.0",
+ FixedVersion: "1.0.1",
+ Severity: "HIGH",
+ })
+ }
+
+ report := vulnerability.TrivyReport{
+ SchemaVersion: 2,
+ ArtifactName: "example/image:latest",
+ ArtifactType: "container_image",
+ Results: []vulnerability.TrivyResults{
+ {
+ Target: "alpine:3.20",
+ Class: "os-pkgs",
+ Type: "alpine",
+ Vulnerabilities: vulns,
+ },
+ },
+ }
+
+ encoder := json.NewEncoder(tmpFile)
+ require.NoError(t, encoder.Encode(report))
+
+ decoded, err := decodeTrivyReportFromFileInternal(tmpFile)
+ require.NoError(t, err)
+ require.NotNil(t, decoded)
+ require.Equal(t, report.ArtifactName, decoded.ArtifactName)
+ require.Len(t, decoded.Results, 1)
+ require.Len(t, decoded.Results[0].Vulnerabilities, vulnCount)
+}
+
+func TestCleanupTrivyLogTempFilesInternal_RemovesFiles(t *testing.T) {
+ stdoutFile, err := os.CreateTemp("", "trivy-stdout-test-*")
+ require.NoError(t, err)
+ stderrFile, err := os.CreateTemp("", "trivy-stderr-test-*")
+ require.NoError(t, err)
+
+ stdoutPath := stdoutFile.Name()
+ stderrPath := stderrFile.Name()
+
+ _, err = stdoutFile.WriteString("stdout")
+ require.NoError(t, err)
+ _, err = stderrFile.WriteString("stderr")
+ require.NoError(t, err)
+
+ cleanupTrivyLogTempFilesInternal(context.Background(), stdoutFile, stderrFile)
+
+ _, err = os.Stat(stdoutPath)
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+
+ _, err = os.Stat(stderrPath)
+ require.Error(t, err)
+ require.True(t, os.IsNotExist(err))
+}