Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: buildkit not using the image exporter (#2469)
Kyle Mendell7a57d1053aed
diff --git a/backend/internal/bootstrap/services_bootstrap.go b/backend/internal/bootstrap/services_bootstrap.goindex 64c3fc581..614a3c4fd 100644--- a/backend/internal/bootstrap/services_bootstrap.go+++ b/backend/internal/bootstrap/services_bootstrap.go@@ -80,7 +80,7 @@ func initializeServices(ctx context.Context, db *database.DB, cfg *config.Configsvcs.ImageUpdate = services.NewImageUpdateService(db, svcs.Settings, svcs.ContainerRegistry, svcs.Docker, svcs.Event, svcs.Notification)svcs.Image = services.NewImageService(db, svcs.Docker, svcs.ContainerRegistry, svcs.ImageUpdate, svcs.Vulnerability, svcs.Event)svcs.GitRepository = services.NewGitRepositoryService(db, cfg.GitWorkDir, svcs.Event, svcs.Settings)- svcs.Build = services.NewBuildService(db, svcs.Settings, svcs.Docker, svcs.ContainerRegistry, svcs.GitRepository)+ svcs.Build = services.NewBuildService(db, svcs.Settings, svcs.Docker, svcs.ContainerRegistry, svcs.GitRepository, svcs.Event)svcs.BuildWorkspace = services.NewBuildWorkspaceService(svcs.Settings)svcs.Project = services.NewProjectService(db, svcs.Settings, svcs.Event, svcs.Image, svcs.Docker, svcs.Build, cfg)svcs.Container = services.NewContainerService(db, svcs.Event, svcs.Docker, svcs.Image, svcs.Settings, svcs.Project)diff --git a/backend/internal/services/build_service.go b/backend/internal/services/build_service.goindex 4b867a109..5c86e4213 100644--- a/backend/internal/services/build_service.go+++ b/backend/internal/services/build_service.go@@ -7,6 +7,7 @@ import ("fmt""io""log/slog"+ "net/url""os""path/filepath""strings"@@ -29,6 +30,7 @@ type BuildService struct {dockerService *DockerClientServiceregistryService *ContainerRegistryServicegitRepository *GitRepositoryService+ eventService *EventServicebuilder buildtypes.BuildergitProbeFn func(context.Context, string, buildgit.AuthConfig) errorgitCloneFn func(context.Context, string, string, buildgit.AuthConfig) (string, error)@@ -43,6 +45,7 @@ func NewBuildService(dockerService *DockerClientService,registryService *ContainerRegistryService,gitRepository *GitRepositoryService,+ eventService *EventService,) *BuildService {svc := &BuildService{db: db,@@ -50,6 +53,7 @@ func NewBuildService(dockerService: dockerService,registryService: registryService,gitRepository: gitRepository,+ eventService: eventService,}svc.builder = libbuild.NewBuilder(svc, dockerService, svc)@@ -136,7 +140,7 @@ func (s *BuildService) BuildImage(ctx context.Context, environmentID string, reqoutputPtr = &output}- provider := req.Provider+ provider := s.effectiveBuildProviderInternal(req.Provider)var digest *stringif result != nil {if result.Provider != "" {@@ -151,17 +155,109 @@ func (s *BuildService) BuildImage(ctx context.Context, environmentID string, reqvar errMsg *stringif err != nil {status = models.ImageBuildStatusFailed- errMsg = new(err.Error())+ errorText := err.Error()+ errMsg = &errorText}- if updateErr := s.completeBuildRecord(ctx, buildRecordID, status, outputPtr, logCapture.Truncated(), errMsg, digest, provider, completedAt, new(completedAt.Sub(startedAt).Milliseconds())); updateErr != nil {+ durationMs := completedAt.Sub(startedAt).Milliseconds()+ if updateErr := s.completeBuildRecord(ctx, buildRecordID, status, outputPtr, logCapture.Truncated(), errMsg, digest, provider, completedAt, &durationMs); updateErr != nil {slog.WarnContext(ctx, "failed to update build history record", "error", updateErr)}}+ if err != nil {+ s.logBuildFailureEventInternal(ctx, environmentID, req, serviceName, buildRecordID, err, user)+ }+return result, err}+func (s *BuildService) logBuildFailureEventInternal(ctx context.Context, environmentID string, req imagetypes.BuildRequest, serviceName, buildRecordID string, err error, user *models.User) {+ if s.eventService == nil || err == nil {+ return+ }++ resourceName := firstNonEmptyStringInternal(req.Tags...)+ if resourceName == "" {+ resourceName = strings.TrimSpace(serviceName)+ }+ if resourceName == "" {+ resourceName = sanitizeBuildContextForEventInternal(req.ContextDir)+ }++ userID := ""+ username := ""+ if user != nil {+ userID = user.ID+ username = user.Username+ }++ metadata := models.JSON{+ "action": "build",+ "provider": s.effectiveBuildProviderInternal(req.Provider),+ "contextDir": sanitizeBuildContextForEventInternal(req.ContextDir),+ "dockerfile": req.Dockerfile,+ "tags": append([]string(nil), req.Tags...),+ }+ if service := strings.TrimSpace(serviceName); service != "" {+ metadata["serviceName"] = service+ }+ if buildRecordID != "" {+ metadata["buildRecordId"] = buildRecordID+ }++ s.eventService.LogErrorEvent(ctx, models.EventTypeImageError, "image", "", resourceName, userID, username, environmentID, err, metadata)+}++func sanitizeBuildContextForEventInternal(raw string) string {+ trimmed := strings.TrimSpace(raw)+ if trimmed == "" {+ return ""+ }++ base, fragment, hasFragment := strings.Cut(trimmed, "#")+ parsed, err := url.Parse(base)+ if err != nil {+ if strings.Contains(base, "@") {+ return "[unparseable URL]"+ }+ return trimmed+ }+ if parsed.User == nil {+ return trimmed+ }++ parsed.User = url.User("redacted")+ sanitized := parsed.String()+ if hasFragment {+ sanitized += "#" + fragment+ }+ return sanitized+}++func (s *BuildService) effectiveBuildProviderInternal(provider string) string {+ provider = strings.ToLower(strings.TrimSpace(provider))+ if provider != "" {+ return provider+ }+ if s.settings != nil {+ provider = strings.ToLower(strings.TrimSpace(s.settings.GetSettingsConfig().BuildProvider.Value))+ }+ if provider == "" {+ return "local"+ }+ return provider+}++func firstNonEmptyStringInternal(values ...string) string {+ for _, value := range values {+ if trimmed := strings.TrimSpace(value); trimmed != "" {+ return trimmed+ }+ }+ return ""+}+func (s *BuildService) resolveBuildRequestInternal(ctx context.Context,req imagetypes.BuildRequest,diff --git a/backend/internal/services/build_service_test.go b/backend/internal/services/build_service_test.goindex 4201ecc53..e39159db0 100644--- a/backend/internal/services/build_service_test.go+++ b/backend/internal/services/build_service_test.go@@ -2,6 +2,7 @@ package servicesimport ("context"+ "errors""io""os""path/filepath"@@ -327,14 +328,79 @@ func TestBuildService_BuildImage_PreservesRemoteSourceInHistory(t *testing.T) {assert.Equal(t, req.ContextDir, record.ContextDir)}+func TestBuildService_BuildImage_FailureRecordsHistoryAndEvent(t *testing.T) {+ db, err := setupBuildHistoryTestDB()+ require.NoError(t, err)++ buildErr := errors.New("BuildKit could not load the image with the docker exporter")+ svc := &BuildService{+ db: db,+ eventService: NewEventService(db, nil, nil),+ builder: testBuildRecorder{+ err: buildErr,+ },+ }+ user := &models.User{+ BaseModel: models.BaseModel{ID: "user-1"},+ Username: "tester",+ }++ req := image.BuildRequest{+ ContextDir: "/builds/demo",+ Dockerfile: "Dockerfile",+ Tags: []string{"arcane.local/demo:test"},+ Provider: "local",+ Load: true,+ }++ _, err = svc.BuildImage(context.Background(), "0", req, nil, "web", user)+ require.ErrorIs(t, err, buildErr)++ var record models.ImageBuild+ require.NoError(t, db.WithContext(context.Background()).First(&record).Error)+ assert.Equal(t, models.ImageBuildStatusFailed, record.Status)+ require.NotNil(t, record.ErrorMessage)+ assert.Contains(t, *record.ErrorMessage, "docker exporter")++ var event models.Event+ require.NoError(t, db.WithContext(context.Background()).First(&event, "type = ?", models.EventTypeImageError).Error)+ assert.Equal(t, models.EventSeverityError, event.Severity)+ require.NotNil(t, event.ResourceName)+ assert.Equal(t, "arcane.local/demo:test", *event.ResourceName)+ assert.Equal(t, "build", event.Metadata["action"])+ assert.Equal(t, "local", event.Metadata["provider"])+ assert.Equal(t, "/builds/demo", event.Metadata["contextDir"])+ assert.Contains(t, event.Metadata["error"], "docker exporter")+ require.NotEmpty(t, event.Metadata["buildRecordId"])+ assert.Equal(t, record.ID, event.Metadata["buildRecordId"])+}++func TestSanitizeBuildContextForEventInternal_RedactsURLCredentials(t *testing.T) {+ assert.Equal(+ t,+ "https://[email protected]/zeroZshadow/secretproject.git#main:app",+ sanitizeBuildContextForEventInternal("https://[email protected]/zeroZshadow/secretproject.git#main:app"),+ )+ assert.Equal(+ t,+ "[unparseable URL]",+ sanitizeBuildContextForEventInternal("https://user:pass@[bad-host/repo.git"),+ )+ assert.Equal(t, "/builds/demo", sanitizeBuildContextForEventInternal("/builds/demo"))+}+type testBuildRecorder struct {onBuild func(image.BuildRequest)+ err error}func (b testBuildRecorder) BuildImage(_ context.Context, req image.BuildRequest, _ io.Writer, _ string) (*image.BuildResult, error) {if b.onBuild != nil {b.onBuild(req)}+ if b.err != nil {+ return nil, b.err+ }return &image.BuildResult{Provider: "local"}, nil}@@ -343,7 +409,7 @@ func setupBuildHistoryTestDB() (*database.DB, error) {if err != nil {return nil, err}- if err := db.AutoMigrate(&models.ImageBuild{}); err != nil {+ if err := db.AutoMigrate(&models.ImageBuild{}, &models.Event{}); err != nil {return nil, err}return &database.DB{DB: db}, nildiff --git a/backend/pkg/libarcane/libbuild/builder_buildkit.go b/backend/pkg/libarcane/libbuild/builder_buildkit.goindex 3b76f1020..a2cce4ffd 100644--- a/backend/pkg/libarcane/libbuild/builder_buildkit.go+++ b/backend/pkg/libarcane/libbuild/builder_buildkit.go@@ -99,7 +99,7 @@ func normalizeEntitlementsInternal(entitlements []string, privileged bool) []strreturn out}-func (b *builder) buildSolveOptInternal(ctx context.Context, req imagetypes.BuildRequest) (buildkit.SolveOpt, <-chan error, func(), error) {+func (b *builder) buildSolveOptInternal(ctx context.Context, req imagetypes.BuildRequest, providerName string) (buildkit.SolveOpt, <-chan error, func(), error) {fsInput, err := prepareBuildFilesystemInputInternal(req)if err != nil {return buildkit.SolveOpt{}, nil, nil, err@@ -157,7 +157,7 @@ func (b *builder) buildSolveOptInternal(ctx context.Context, req imagetypes.Builvar loadErrCh chan errorexports := make([]buildkit.ExportEntry, 0, 2)- if req.Push {+ if req.Push && providerName != "local" {exports = append(exports, buildkit.ExportEntry{Type: "image",Attrs: map[string]string{@@ -167,7 +167,20 @@ func (b *builder) buildSolveOptInternal(ctx context.Context, req imagetypes.Buil},})}- if req.Load {++ if providerName == "local" && (req.Load || req.Push) {+ attrs := map[string]string{+ "name": strings.Join(req.Tags, ","),+ "oci-mediatypes": "true",+ }+ if req.Push {+ attrs["push"] = "true"+ }+ exports = append(exports, buildkit.ExportEntry{+ Type: "image",+ Attrs: attrs,+ })+ } else if req.Load {exportEntry, errCh, err := b.buildLoadExportInternal(ctx, req.Tags)if err != nil {cleanup()diff --git a/backend/pkg/libarcane/libbuild/builder_buildkit_test.go b/backend/pkg/libarcane/libbuild/builder_buildkit_test.goindex 7f65ce155..19fb164fb 100644--- a/backend/pkg/libarcane/libbuild/builder_buildkit_test.go+++ b/backend/pkg/libarcane/libbuild/builder_buildkit_test.go@@ -2,11 +2,15 @@ package libbuildimport ("context"+ "io"+ "net/http"+ "net/http/httptest""os""path/filepath""testing"imagetypes "github.com/getarcaneapp/arcane/types/image"+ dockerclient "github.com/moby/moby/client""github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -24,7 +28,7 @@ func TestBuildSolveOptInternal_StagesInlineDockerfile(t *testing.T) {},}- solveOpt, loadErrCh, cleanup, err := b.buildSolveOptInternal(context.Background(), req)+ solveOpt, loadErrCh, cleanup, err := b.buildSolveOptInternal(context.Background(), req, "local")require.NoError(t, err)defer cleanup()assert.Nil(t, loadErrCh)@@ -43,3 +47,97 @@ func TestBuildSolveOptInternal_StagesInlineDockerfile(t *testing.T) {require.NoError(t, err)assert.Equal(t, "hello\n", string(appContents))}++func TestBuildSolveOptInternal_LocalLoadUsesImageExporter(t *testing.T) {+ contextDir := createBuildkitTestContext(t)+ b := &builder{}++ solveOpt, loadErrCh, cleanup, err := b.buildSolveOptInternal(context.Background(), imagetypes.BuildRequest{+ ContextDir: contextDir,+ Dockerfile: "Dockerfile",+ Tags: []string{"arcane.local/app:test"},+ Load: true,+ }, "local")+ require.NoError(t, err)+ defer cleanup()++ require.Nil(t, loadErrCh)+ require.Len(t, solveOpt.Exports, 1)+ assert.Equal(t, "image", solveOpt.Exports[0].Type)+ assert.Equal(t, "arcane.local/app:test", solveOpt.Exports[0].Attrs["name"])+ assert.NotContains(t, solveOpt.Exports[0].Attrs, "push")+ assert.Nil(t, solveOpt.Exports[0].Output)+}++func TestBuildSolveOptInternal_LocalPushAndLoadUsesSingleImageExporter(t *testing.T) {+ contextDir := createBuildkitTestContext(t)+ b := &builder{}++ solveOpt, loadErrCh, cleanup, err := b.buildSolveOptInternal(context.Background(), imagetypes.BuildRequest{+ ContextDir: contextDir,+ Dockerfile: "Dockerfile",+ Tags: []string{"registry.example.com/app:test"},+ Push: true,+ Load: true,+ }, "local")+ require.NoError(t, err)+ defer cleanup()++ require.Nil(t, loadErrCh)+ require.Len(t, solveOpt.Exports, 1)+ assert.Equal(t, "image", solveOpt.Exports[0].Type)+ assert.Equal(t, "registry.example.com/app:test", solveOpt.Exports[0].Attrs["name"])+ assert.Equal(t, "true", solveOpt.Exports[0].Attrs["push"])+}++func TestBuildSolveOptInternal_NonLocalLoadKeepsDockerExporter(t *testing.T) {+ contextDir := createBuildkitTestContext(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/v1.54/images/load" {+ http.NotFound(w, r)+ return+ }+ _, _ = io.Copy(io.Discard, r.Body)+ _ = r.Body.Close()+ _, _ = w.Write([]byte("{}\n"))+ }))+ defer server.Close()++ client, err := dockerclient.NewClientWithOpts(dockerclient.WithHost(server.URL), dockerclient.WithVersion("1.54"))+ require.NoError(t, err)++ b := &builder{dockerClientProvider: testDockerClientProvider{client: client}}+ solveOpt, loadErrCh, cleanup, err := b.buildSolveOptInternal(context.Background(), imagetypes.BuildRequest{+ ContextDir: contextDir,+ Dockerfile: "Dockerfile",+ Tags: []string{"arcane.local/app:test"},+ Load: true,+ }, "depot")+ require.NoError(t, err)+ defer cleanup()++ require.NotNil(t, loadErrCh)+ require.Len(t, solveOpt.Exports, 1)+ assert.Equal(t, "docker", solveOpt.Exports[0].Type)+ require.NotNil(t, solveOpt.Exports[0].Output)++ output, err := solveOpt.Exports[0].Output(nil)+ require.NoError(t, err)+ require.NoError(t, output.Close())+ require.NoError(t, <-loadErrCh)+}++func createBuildkitTestContext(t *testing.T) string {+ t.Helper()+ contextDir := t.TempDir()+ require.NoError(t, os.WriteFile(filepath.Join(contextDir, "Dockerfile"), []byte("FROM alpine:3.20\n"), 0o644))+ return contextDir+}++type testDockerClientProvider struct {+ client *dockerclient.Client+}++func (p testDockerClientProvider) GetClient(context.Context) (*dockerclient.Client, error) {+ return p.client, nil+}diff --git a/backend/pkg/libarcane/libbuild/builder_core.go b/backend/pkg/libarcane/libbuild/builder_core.goindex 25de3658b..aa0df1d4a 100644--- a/backend/pkg/libarcane/libbuild/builder_core.go+++ b/backend/pkg/libarcane/libbuild/builder_core.go@@ -110,7 +110,7 @@ func (b *builder) buildWithBuildkitSessionInternal(}}()- solveOpt, loadErrCh, cleanupSolveOpt, err := b.buildSolveOptInternal(ctx, req)+ solveOpt, loadErrCh, cleanupSolveOpt, err := b.buildSolveOptInternal(ctx, req, providerName)if err != nil {buildErr = errreturn nil, err@@ -136,9 +136,10 @@ func (b *builder) buildWithBuildkitSessionInternal(})resp, err := session.Client.Solve(ctx, nil, solveOpt, statusCh)- buildErr = errif err != nil {+ err = wrapBuildkitSolveErrorInternal(err, providerName)+ buildErr = errwriteProgressEventInternal(progressWriter, imagetypes.ProgressEvent{Type: "build",Service: serviceName,@@ -184,6 +185,18 @@ func (b *builder) buildWithBuildkitSessionInternal(}, nil}+func wrapBuildkitSolveErrorInternal(err error, providerName string) error {+ if err == nil {+ return nil+ }++ if strings.Contains(err.Error(), `exporter "docker" could not be found`) {+ return fmt.Errorf("BuildKit could not load the image with the docker exporter for provider %s; the builder does not expose that exporter: %w", providerName, err)+ }++ return err+}+func buildkitAuthConfigProviderInternal(defaultProvider authprovider.AuthConfigProvider, registryAuthProvider buildtypes.RegistryAuthProvider) authprovider.AuthConfigProvider {return func(ctx context.Context, host string, scope []string, cacheCheck authprovider.ExpireCachedAuthCheck) (configtypes.AuthConfig, error) {if registryAuthProvider != nil {diff --git a/backend/pkg/libarcane/libbuild/builder_core_test.go b/backend/pkg/libarcane/libbuild/builder_core_test.goindex cd4d17ca8..285ac027f 100644--- a/backend/pkg/libarcane/libbuild/builder_core_test.go+++ b/backend/pkg/libarcane/libbuild/builder_core_test.go@@ -98,3 +98,12 @@ func TestBuildkitAuthConfigProvider_FallsBackToDefaultProvider(t *testing.T) {assert.Equal(t, defaultCfg, cfg)})}++func TestWrapBuildkitSolveErrorInternal_LeavesGenericErrorsUnchanged(t *testing.T) {+ err := errors.New("failed to solve: Dockerfile parse error on line 3")++ wrapped := wrapBuildkitSolveErrorInternal(err, "local")++ require.ErrorIs(t, wrapped, err)+ assert.Equal(t, err.Error(), wrapped.Error())+}