Skip to content

Modern Docker Management, Designed for Everyone

fix: walk project workspace level-by-level and show truncation alert (#4479)

Kyle Mendellecc20dfc4673
diff --git a/backend/pkg/projects/workspace.go b/backend/pkg/projects/workspace.go
index 9373dbe82..4ab6c467b 100644
--- a/backend/pkg/projects/workspace.go
+++ b/backend/pkg/projects/workspace.go
@@ -72,9 +72,9 @@ func ReadProjectWorkspace(
projectAbs = filepath.Clean(projectAbs)
// The workspace tree walk and apply engine stay on os.Root rather than
- // acfs: the revision hash depends on fs.WalkDir's lexical visit order, and
- // acfs walks in a different order — switching would change every revision
- // and mass-invalidate open editor drafts with false 409s.
+ // acfs: the revision hash depends on the level-by-level, name-sorted visit
+ // order, and acfs walks in a different order — switching would change every
+ // revision and mass-invalidate open editor drafts with false 409s.
root, err := os.OpenRoot(projectAbs)
if err != nil {
return nil, "", false, fmt.Errorf("open project directory: %w", err)
@@ -93,8 +93,8 @@ func ReadProjectWorkspace(
revisionHash: sha256.New(),
}
- if walkDirErr := fs.WalkDir(root.FS(), ".", walker.visit); walkDirErr != nil {
- return nil, "", false, walkDirErr
+ if walkErr := walker.walk(root.FS()); walkErr != nil {
+ return nil, "", false, walkErr
}
slices.SortFunc(walker.files, func(a, b workspacetypes.FileEntry) int {
@@ -121,57 +121,73 @@ type projectWorkspaceTreeWalkerInternal struct {
truncated bool
}
-func (w *projectWorkspaceTreeWalkerInternal) visit(rel string, entry fs.DirEntry, walkErr error) error {
- if walkErr != nil {
- // fs.WalkDir reports a directory it could not read by invoking the
- // callback a second time with the ReadDir error. Skipping keeps the
- // rest of the tree intact instead of discarding every sibling; the
- // directory itself was already recorded by the first call. A failure
- // on the project workspace root, or one that is not a permission/disappearance
- // error, stays fatal so real I/O problems remain observable.
- if rel != "." && entry != nil && entry.IsDir() && (os.IsPermission(walkErr) || os.IsNotExist(walkErr)) {
- slog.DebugContext(w.ctx, "Skipping unreadable project workspace subdirectory", "relativePath", rel, "error", walkErr)
- return fs.SkipDir
+// walk visits the tree level by level (every top-level entry, then the next
+// level down) so a large early subtree cannot use the whole entry budget
+// before later top-level siblings are seen (#4452).
+func (w *projectWorkspaceTreeWalkerInternal) walk(fsys fs.FS) error {
+ for queue := []string{"."}; len(queue) > 0; queue = queue[1:] {
+ entries, err := fs.ReadDir(fsys, queue[0])
+ if err != nil {
+ // An unreadable subdirectory is skipped so the rest of the tree stays
+ // intact; it was already recorded when visited from its parent. A
+ // failure on the root, or one that is not a permission/disappearance
+ // error, stays fatal so real I/O problems remain observable.
+ if queue[0] != "." && (os.IsPermission(err) || os.IsNotExist(err)) {
+ slog.DebugContext(w.ctx, "Skipping unreadable project workspace subdirectory", "relativePath", queue[0], "error", err)
+ continue
+ }
+ return err
+ }
+ for _, entry := range entries {
+ rel := path.Join(queue[0], entry.Name())
+ descend, visitErr := w.visit(rel, entry)
+ if visitErr != nil {
+ return visitErr
+ }
+ if w.truncated {
+ return nil
+ }
+ if descend {
+ queue = append(queue, rel)
+ }
}
- return walkErr
- }
- if rel == "." {
- return nil
}
+ return nil
+}
- // The walk visits entries in deterministic lexical order, so cutting
- // off after maxEntries still yields a stable revision as long as the
- // concurrency compare walk uses the same cap.
+// visit records one entry and reports whether the walk should descend into it.
+func (w *projectWorkspaceTreeWalkerInternal) visit(rel string, entry fs.DirEntry) (bool, error) {
+ // Entries are visited in a fixed level-by-level, name-sorted order, so
+ // cutting off after maxEntries still yields a stable revision as long as
+ // the concurrency compare walk uses the same cap.
if w.entryCount >= w.maxEntries {
w.truncated = true
- return fs.SkipAll
+ return false, nil
}
depth := strings.Count(rel, "/") + 1
if depth > w.maxDepth {
- return kit.Ternary(entry.IsDir(), fs.SkipDir, nil)
+ return false, nil
}
if entry.IsDir() && w.skipDirs[entry.Name()] {
- return fs.SkipDir
+ return false, nil
}
info, err := entry.Info()
if err != nil {
// An entry that vanished mid-walk or that we are not allowed to stat is
// skipped rather than failing the whole tree; anything else surfaces.
- // fs.SkipDir on a non-directory would break out of the parent's sibling
- // loop, so split on IsDir.
if !os.IsPermission(err) && !os.IsNotExist(err) {
- return err
+ return false, err
}
slog.DebugContext(w.ctx, "Skipping unreadable project workspace entry", "relativePath", rel, "error", err)
- return kit.Ternary(entry.IsDir(), fs.SkipDir, nil)
+ return false, nil
}
isProtected := w.protected[rel]
if isProtected {
- return nil
+ return entry.IsDir(), nil
}
// Protected Compose configuration is intentionally absent from both the
@@ -223,7 +239,7 @@ func (w *projectWorkspaceTreeWalkerInternal) visit(rel string, entry fs.DirEntry
}
w.files = append(w.files, workspaceEntry)
- return nil
+ return entry.IsDir(), nil
}
func classifyProjectWorkspaceFileInternal(filePath string, size, maxFileSizeBytes int64) (bool, string) {
diff --git a/backend/pkg/projects/workspace_test.go b/backend/pkg/projects/workspace_test.go
index 43c3eb92e..bd319cb26 100644
--- a/backend/pkg/projects/workspace_test.go
+++ b/backend/pkg/projects/workspace_test.go
@@ -4,9 +4,11 @@ import (
"fmt"
"os"
"path/filepath"
+ "slices"
"testing"
"github.com/getarcaneapp/arcane/types/v2/project"
+ "github.com/getarcaneapp/arcane/types/v2/workspace"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
kit "go.getarcane.app/kit/pkg"
@@ -350,14 +352,19 @@ func TestReadProjectWorkspace_CapsEntriesWithStableRevision(t *testing.T) {
projectDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(projectDir, "compose.yaml"), []byte("services: {}\n"), 0o644))
+ require.NoError(t, os.Mkdir(filepath.Join(projectDir, "a-large"), 0o755))
for i := range 5 {
- require.NoError(t, os.WriteFile(filepath.Join(projectDir, fmt.Sprintf("file-%d.txt", i)), []byte("x\n"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectDir, "a-large", fmt.Sprintf("file-%d.txt", i)), []byte("x\n"), 0o644))
}
+ require.NoError(t, os.WriteFile(filepath.Join(projectDir, "z-later.txt"), []byte("x\n"), 0o644))
files, revision, truncated, err := ReadProjectWorkspace(t.Context(), projectDir, 3, "", "compose.yaml", 3, 0)
require.NoError(t, err)
assert.True(t, truncated)
assert.Len(t, files, 3)
+ assert.True(t, slices.ContainsFunc(files, func(entry workspace.FileEntry) bool {
+ return entry.RelativePath == "z-later.txt"
+ }), "a large early subtree must not hide later root-level siblings")
_, again, truncatedAgain, err := ReadProjectWorkspace(t.Context(), projectDir, 3, "", "compose.yaml", 3, 0)
require.NoError(t, err)
diff --git a/frontend/src/routes/(app)/projects/[projectId]/+page.svelte b/frontend/src/routes/(app)/projects/[projectId]/+page.svelte
index 41a615a74..8f45a6c0c 100644
--- a/frontend/src/routes/(app)/projects/[projectId]/+page.svelte
+++ b/frontend/src/routes/(app)/projects/[projectId]/+page.svelte
@@ -1552,6 +1552,13 @@
{/if}
</div>
+ {#if layoutMode === 'tree' && projectWorkspaceQuery.data?.fileTreeTruncated}
+ <Alert.Root class="mb-3 shrink-0">
+ <AlertIcon class="size-4" />
+ <Alert.Description>{m.volumes_workspace_truncated()}</Alert.Description>
+ </Alert.Root>
+ {/if}
+
<div class="min-h-0 flex-1">
{#if layoutMode === 'tree'}
<div class="flex h-full min-h-0 flex-col overflow-hidden rounded-lg border border-border bg-card">