Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
fix(flows): return partial container listings instead of failing; stop the refetch skeleton flash
Sergey Kozyrenko7b67b1c0ea81
diff --git a/backend/pkg/docker/client.go b/backend/pkg/docker/client.goindex 18ac9862..5eb7b268 100644--- a/backend/pkg/docker/client.go+++ b/backend/pkg/docker/client.go@@ -597,6 +597,9 @@ type ContainerEntryError struct {type ContainerDirListing struct {Files []container.PathStatFailures []ContainerEntryError+ // Truncated is set when the directory held more than maxListEntries children+ // and only the first maxListEntries were listed.+ Truncated bool}func (dc *dockerClient) ListContainerDir(@@ -660,25 +663,27 @@ func (dc *dockerClient) ListContainerDir(entryPaths = append(entryPaths, tok)}+ // Over the cap, list the first maxListEntries and flag the truncation rather+ // than failing the whole listing — the caller reports the partiality.+ truncated := falseif len(entryPaths) > maxListEntries {- return ContainerDirListing{}, fmt.Errorf("container directory '%s' has too many entries (%d, limit %d) — narrow the path", dirPath, len(entryPaths), maxListEntries)+ entryPaths = entryPaths[:maxListEntries]+ truncated = true}stats, failures := statContainerEntries(ctx, entryPaths, containerListWorkers, func(ctx context.Context, entryPath string) (container.PathStat, error) {return dc.ContainerStatPath(ctx, containerID, entryPath)})- // A cancelled context, or a listing where every entry failed, is a directory-- // level fault (client gone, or the container/daemon died mid fan-out) — not a- // partial listing. Surface it as an error rather than a misleading empty 200.+ // A cancelled context is a directory-level fault (the client is gone), not a+ // partial listing, so surface it as an error. Entries that individually failed+ // to stat are carried in Failures — the find exec already proved the container+ // alive, so a live directory degrades rather than 500s even if every entry failed.if err := ctx.Err(); err != nil {return ContainerDirListing{}, fmt.Errorf("listing container directory '%s': %w", dirPath, err)}- if len(entryPaths) > 0 && len(stats) == 0 {- return ContainerDirListing{}, fmt.Errorf("failed to read any of the %d entries in container directory '%s' (container may be gone): %w", len(entryPaths), dirPath, failures[0].err)- }- listing := ContainerDirListing{Files: stats}+ listing := ContainerDirListing{Files: stats, Truncated: truncated}for _, f := range failures {listing.Failures = append(listing.Failures, ContainerEntryError{Name: path.Base(f.name),diff --git a/backend/pkg/server/docs/docs.go b/backend/pkg/server/docs/docs.goindex cd477efe..40cc49e4 100644--- a/backend/pkg/server/docs/docs.go+++ b/backend/pkg/server/docs/docs.go@@ -8374,6 +8374,10 @@ const docTemplate = `{},"total": {"type": "integer"+ },+ "truncated": {+ "description": "Truncated is true when the directory held more entries than the listing cap\nand only the first page was returned.",+ "type": "boolean"}}},diff --git a/backend/pkg/server/docs/swagger.json b/backend/pkg/server/docs/swagger.jsonindex 989a8805..ea078cd2 100644--- a/backend/pkg/server/docs/swagger.json+++ b/backend/pkg/server/docs/swagger.json@@ -8366,6 +8366,10 @@},"total": {"type": "integer"+ },+ "truncated": {+ "description": "Truncated is true when the directory held more entries than the listing cap\nand only the first page was returned.",+ "type": "boolean"}}},diff --git a/backend/pkg/server/docs/swagger.yaml b/backend/pkg/server/docs/swagger.yamlindex 3f2172a5..78dee0bf 100644--- a/backend/pkg/server/docs/swagger.yaml+++ b/backend/pkg/server/docs/swagger.yaml@@ -465,6 +465,11 @@ definitions:type: stringtotal:type: integer+ truncated:+ description: |-+ Truncated is true when the directory held more entries than the listing cap+ and only the first page was returned.+ type: booleantype: objectmodels.CopyResourceRequest:properties:diff --git a/backend/pkg/server/models/flow_files.go b/backend/pkg/server/models/flow_files.goindex e478ff70..44a40b96 100644--- a/backend/pkg/server/models/flow_files.go+++ b/backend/pkg/server/models/flow_files.go@@ -43,6 +43,9 @@ type ContainerFiles struct {Files []ContainerFile `json:"files"`Failures []ContainerFileError `json:"failures,omitempty"`Total uint64 `json:"total"`+ // Truncated is true when the directory held more entries than the listing cap+ // and only the first page was returned.+ Truncated bool `json:"truncated,omitempty"`}// PullFlowFilesRequest is the request body for pulling files from a container.diff --git a/backend/pkg/server/services/flow_files.go b/backend/pkg/server/services/flow_files.goindex cae13891..7f1a434e 100644--- a/backend/pkg/server/services/flow_files.go+++ b/backend/pkg/server/services/flow_files.go@@ -987,6 +987,7 @@ func (s *FlowFileService) GetFlowContainerFiles(c *gin.Context) {// EVERY path fails at the directory level, nothing could be listed and the// request is failed as a whole (checked after the loop).pathsListed := 0+ truncated := falsevar firstPathErr errorrecordPathFailure := func(p string, err error) {if firstPathErr == nil {@@ -1035,6 +1036,9 @@ func (s *FlowFileService) GetFlowContainerFiles(c *gin.Context) {continue}pathsListed+++ if listing.Truncated {+ truncated = true+ }for _, stat := range listing.Files {file := convertContainerFile(containerPath, stat)if _, seen := seenPaths[file.Path]; !seen {@@ -1128,10 +1132,11 @@ func (s *FlowFileService) GetFlowContainerFiles(c *gin.Context) {}response.Success(c, http.StatusOK, models.ContainerFiles{- Path: responsePath,- Files: allFiles,- Failures: allFailures,- Total: uint64(len(allFiles)),+ Path: responsePath,+ Files: allFiles,+ Failures: allFailures,+ Total: uint64(len(allFiles)),+ Truncated: truncated,})}diff --git a/backend/pkg/server/services/flow_files_partial_test.go b/backend/pkg/server/services/flow_files_partial_test.goindex 77be6fdb..ceba24c6 100644--- a/backend/pkg/server/services/flow_files_partial_test.go+++ b/backend/pkg/server/services/flow_files_partial_test.go@@ -102,3 +102,33 @@ func TestGetFlowContainerFiles_PathNeverInBothArrays(t *testing.T) {"order %q: /work/x must NOT also be in Failures", order)}}++// A live container whose entries all fail to stat degrades to a 200 partial+// listing (empty files, entries in Failures), not a 500.+func TestGetFlowContainerFiles_AllEntriesFailedStillReturns200(t *testing.T) {+ fake := &fakeDockerClient{running: true}+ fake.statPathMap = map[string]container.PathStat{"/work": {Mode: os.ModeDir | 0755}}+ fake.listDirMap = map[string][]container.PathStat{"/work": {}}+ fake.listDirFailMap = map[string][]docker.ContainerEntryError{+ "/work": {+ {Name: "a", Path: "/work/a", Err: fmt.Errorf("stat: gone")},+ {Name: "b", Path: "/work/b", Err: fmt.Errorf("stat: gone")},+ },+ }+ code, resp := listContainerFiles(t, fake, "paths[]=/work")+ require.Equal(t, http.StatusOK, *code)+ assert.Empty(t, resp.Files)+ assert.Len(t, resp.Failures, 2)+}++// A truncated listing surfaces Truncated so the UI can warn the user they are not+// seeing every entry.+func TestGetFlowContainerFiles_TruncatedFlagSurfaced(t *testing.T) {+ fake := &fakeDockerClient{running: true}+ fake.statPathMap = map[string]container.PathStat{"/big": {Mode: os.ModeDir | 0755}}+ fake.listDirMap = map[string][]container.PathStat{"/big": {{Name: "f", Mode: 0644, Size: 1}}}+ fake.listDirTruncated = map[string]bool{"/big": true}+ code, resp := listContainerFiles(t, fake, "paths[]=/big")+ require.Equal(t, http.StatusOK, *code)+ assert.True(t, resp.Truncated)+}diff --git a/backend/pkg/server/services/flow_files_test.go b/backend/pkg/server/services/flow_files_test.goindex 307fa42b..6044d2b9 100644--- a/backend/pkg/server/services/flow_files_test.go+++ b/backend/pkg/server/services/flow_files_test.go@@ -968,11 +968,12 @@ type fakeDockerClient struct {// Per-path overrides for multi-path tests; take precedence over the// single-value fields above when the queried path has an entry here.- statPathMap map[string]container.PathStat- statPathErrMap map[string]error- listDirMap map[string][]container.PathStat- listDirFailMap map[string][]docker.ContainerEntryError- listDirErrMap map[string]error+ statPathMap map[string]container.PathStat+ statPathErrMap map[string]error+ listDirMap map[string][]container.PathStat+ listDirFailMap map[string][]docker.ContainerEntryError+ listDirErrMap map[string]error+ listDirTruncated map[string]bool// CopyFromContainer behaviour.copyFromBody []byte@@ -1058,12 +1059,12 @@ func (f *fakeDockerClient) ListContainerDir(_ context.Context, _ string, p strin}if f.listDirFailMap != nil {if fails, ok := f.listDirFailMap[p]; ok {- return docker.ContainerDirListing{Files: f.listDirMap[p], Failures: fails}, nil+ return docker.ContainerDirListing{Files: f.listDirMap[p], Failures: fails, Truncated: f.listDirTruncated[p]}, nil}}if f.listDirMap != nil {if dir, ok := f.listDirMap[p]; ok {- return docker.ContainerDirListing{Files: dir}, nil+ return docker.ContainerDirListing{Files: dir, Truncated: f.listDirTruncated[p]}, nil}}return docker.ContainerDirListing{Files: f.listDir}, f.listDirErrdiff --git a/frontend/src/features/flows/files/flow-files-pull-dialog.tsx b/frontend/src/features/flows/files/flow-files-pull-dialog.tsxindex bde9f929..1aacc92a 100644--- a/frontend/src/features/flows/files/flow-files-pull-dialog.tsx+++ b/frontend/src/features/flows/files/flow-files-pull-dialog.tsx@@ -138,6 +138,7 @@ function FlowFilesPullDialogForm({ cachedFiles, flowId, onClose, onSuccess }: Flfiles,isLoading: isListingLoading,refetch: refetchListing,+ truncated: isListingTruncated,} = useFlowContainerFiles({ flowId, paths: listingPaths });/**@@ -468,6 +469,17 @@ function FlowFilesPullDialogForm({ cachedFiles, flowId, onClose, onSuccess }: Fl</Alert>)}+ {isListingTruncated && (+ <Alert>+ <TriangleAlert />+ <AlertTitle>Directory truncated</AlertTitle>+ <AlertDescription>+ This directory has too many entries to list in full; only the first {files.length} are+ shown. Open a subfolder to see the rest.+ </AlertDescription>+ </Alert>+ )}+<FileManagerbulkActions={bulkActions}className="h-[360px]"diff --git a/frontend/src/features/flows/files/flow-files-utils.ts b/frontend/src/features/flows/files/flow-files-utils.tsindex 5b3c7bf8..314f063c 100644--- a/frontend/src/features/flows/files/flow-files-utils.ts+++ b/frontend/src/features/flows/files/flow-files-utils.ts@@ -21,6 +21,8 @@ export interface ContainerFilesResponse {files: RestContainerFile[];path: string;total: number;+ /** True when the directory held more entries than the cap and only the first page was returned. */+ truncated?: boolean;}export type FlowFile = FlowFileFragmentFragment;diff --git a/frontend/src/features/flows/files/use-flow-container-files.ts b/frontend/src/features/flows/files/use-flow-container-files.tsindex dd713cca..dcfcad60 100644--- a/frontend/src/features/flows/files/use-flow-container-files.ts+++ b/frontend/src/features/flows/files/use-flow-container-files.ts@@ -36,6 +36,8 @@ interface UseFlowContainerFilesResult {* the dialog.*/refetch: () => Promise<void>;+ /** True when the directory had more entries than the cap and only the first page was returned. */+ truncated: boolean;}/**@@ -54,6 +56,7 @@ interface UseFlowContainerFilesResult {export function useFlowContainerFiles({ flowId, paths }: UseFlowContainerFilesParams): UseFlowContainerFilesResult {const [files, setFiles] = useState<FileNode[]>([]);const [failures, setFailures] = useState<ContainerFileFailure[]>([]);+ const [truncated, setTruncated] = useState(false);const [isLoading, setIsLoading] = useState(false);const [error, setError] = useState<Error | null>(null);@@ -69,11 +72,18 @@ export function useFlowContainerFiles({ flowId, paths }: UseFlowContainerFilesPa// when the user drills in / out faster than the server can respond).const currentTokenRef = useRef(0);+ // The path key of the last fetch. A fetch clears the visible listing only when+ // the path actually changes, so an in-place refetch (Refresh / post-Pull) keeps+ // the current rows under the loading guard instead of flashing a skeleton.+ const lastPathsKeyRef = useRef<null | string>(null);+const fetchListing = useCallback(async () => {if (!flowId || paths.length === 0) {currentTokenRef.current += 1;+ lastPathsKeyRef.current = null;setFiles([]);setFailures([]);+ setTruncated(false);setIsLoading(false);setError(null);@@ -84,10 +94,17 @@ export function useFlowContainerFiles({ flowId, paths }: UseFlowContainerFilesPasetIsLoading(true);setError(null);- // Clear the previous directory's data so a pending navigation never shows- // the old listing / old failure banner attributed to the new breadcrumb.- setFiles([]);- setFailures([]);++ // Only reset the listing when navigating to a different path, so a pending+ // navigation never shows the old listing under the new breadcrumb — a+ // same-path refetch keeps the current rows visible.+ if (pathsKey !== lastPathsKeyRef.current) {+ setFiles([]);+ setFailures([]);+ setTruncated(false);+ }++ lastPathsKeyRef.current = pathsKey;try {const url = `${FLOW_FILES_CONTAINER_API_PATH(flowId)}?${buildPathsQuery(paths)}`;@@ -103,6 +120,7 @@ export function useFlowContainerFiles({ flowId, paths }: UseFlowContainerFilesPasetFiles(data.files.map(containerFileToFileNode));setFailures(data.failures ?? []);+ setTruncated(data.truncated ?? false);} catch (caught) {if (token !== currentTokenRef.current) {return;@@ -111,12 +129,13 @@ export function useFlowContainerFiles({ flowId, paths }: UseFlowContainerFilesPasetError(new Error(getApiErrorMessage(caught, 'Failed to load container files')));setFiles([]);setFailures([]);+ setTruncated(false);} finally {if (token === currentTokenRef.current) {setIsLoading(false);}}- }, [flowId, paths]);+ }, [flowId, paths, pathsKey]);useEffect(() => {// fetchListing is an async callback that handles its own loading state via setState@@ -135,5 +154,6 @@ export function useFlowContainerFiles({ flowId, paths }: UseFlowContainerFilesPafiles,isLoading,refetch: fetchListing,+ truncated,};}