Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

fix: update resource handling and API documentation

Dmitry Ng8830ae70107d
diff --git a/backend/pkg/server/docs/docs.go b/backend/pkg/server/docs/docs.go
index 19c8beae..daef6312 100644
--- a/backend/pkg/server/docs/docs.go
+++ b/backend/pkg/server/docs/docs.go
@@ -2038,7 +2038,7 @@ const docTemplate = `{
},
{
"type": "string",
- "description": "relative path in cache: uploads/\u003cname\u003e or container/\u003cname\u003e",
+ "description": "relative path in cache: uploads/, resources/, or container/",
"name": "path",
"in": "query",
"required": true
@@ -2193,7 +2193,7 @@ const docTemplate = `{
},
{
"type": "string",
- "description": "relative path in cache: uploads/\u003cname\u003e or container/\u003cname\u003e",
+ "description": "relative path in cache: uploads/, resources/, or container/",
"name": "path",
"in": "query",
"required": true
@@ -2328,7 +2328,7 @@ const docTemplate = `{
"BearerAuth": []
}
],
- "description": "Copies one or more user resources (identified by ID) into flow-{id}-data/resources/.\nFiles already present in the flow are skipped unless force=true, in which case they are replaced.",
+ "description": "Copies one or more user resources (identified by ID) into flow-{id}-data/resources/.\nFiles already present in the flow are skipped unless force=true, in which case they are replaced.\nWhen the primary container for this flow is running, new or replaced files are pushed to /work/resources/ (best-effort), same as uploads.",
"consumes": [
"application/json"
],
@@ -2417,7 +2417,6 @@ const docTemplate = `{
"BearerAuth": []
}
],
- "description": "Reads a file from flow-{id}-data/{sourcePath}, computes MD5, stores the blob, inserts a user_resources row.",
"consumes": [
"application/json"
],
@@ -2427,7 +2426,7 @@ const docTemplate = `{
"tags": [
"FlowFiles"
],
- "summary": "Promote a flow file to user resources",
+ "summary": "Promote a flow file or directory to user resources",
"parameters": [
{
"minimum": 0,
@@ -2449,7 +2448,7 @@ const docTemplate = `{
],
"responses": {
"200": {
- "description": "resource created or updated",
+ "description": "resources created or updated",
"schema": {
"allOf": [
{
@@ -2459,7 +2458,7 @@ const docTemplate = `{
"type": "object",
"properties": {
"data": {
- "$ref": "#/definitions/models.ResourceEntry"
+ "$ref": "#/definitions/models.ResourceList"
}
}
}
@@ -2479,7 +2478,7 @@ const docTemplate = `{
}
},
"404": {
- "description": "source file or flow not found",
+ "description": "source or flow not found",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
@@ -6894,7 +6893,7 @@ const docTemplate = `{
"type": "object",
"required": [
"destination",
- "sourcePath"
+ "source"
],
"properties": {
"destination": {
@@ -6905,8 +6904,8 @@ const docTemplate = `{
"description": "Force overwrites an existing resource at Destination if one already exists.",
"type": "boolean"
},
- "sourcePath": {
- "description": "SourcePath is a relative path within the flow cache, e.g. \"container/work/result.md\" or \"uploads/task.md\".",
+ "source": {
+ "description": "Source is a relative path within the flow cache, e.g. \"container/work/result.md\" or \"uploads/task.md\" or \"uploads/work/\".",
"type": "string"
}
}
@@ -6926,7 +6925,7 @@ const docTemplate = `{
"type": "array",
"minItems": 1,
"items": {
- "type": "string"
+ "type": "integer"
}
}
}
@@ -7251,10 +7250,10 @@ const docTemplate = `{
"id": {
"type": "string"
},
- "isDir": {
+ "is_dir": {
"type": "boolean"
},
- "modifiedAt": {
+ "modified_at": {
"type": "string"
},
"name": {
@@ -7526,10 +7525,10 @@ const docTemplate = `{
"id": {
"type": "string"
},
- "isDir": {
+ "is_dir": {
"type": "boolean"
},
- "modifiedAt": {
+ "modified_at": {
"type": "string"
},
"name": {
@@ -8069,13 +8068,13 @@ const docTemplate = `{
"models.ResourceEntry": {
"type": "object",
"properties": {
- "createdAt": {
+ "created_at": {
"type": "string"
},
"id": {
"type": "integer"
},
- "isDir": {
+ "is_dir": {
"type": "boolean"
},
"name": {
@@ -8087,10 +8086,10 @@ const docTemplate = `{
"size": {
"type": "integer"
},
- "updatedAt": {
+ "updated_at": {
"type": "string"
},
- "userId": {
+ "user_id": {
"type": "integer"
}
}
diff --git a/backend/pkg/server/docs/swagger.json b/backend/pkg/server/docs/swagger.json
index 166e75df..3df9f1a8 100644
--- a/backend/pkg/server/docs/swagger.json
+++ b/backend/pkg/server/docs/swagger.json
@@ -2030,7 +2030,7 @@
},
{
"type": "string",
- "description": "relative path in cache: uploads/\u003cname\u003e or container/\u003cname\u003e",
+ "description": "relative path in cache: uploads/, resources/, or container/",
"name": "path",
"in": "query",
"required": true
@@ -2185,7 +2185,7 @@
},
{
"type": "string",
- "description": "relative path in cache: uploads/\u003cname\u003e or container/\u003cname\u003e",
+ "description": "relative path in cache: uploads/, resources/, or container/",
"name": "path",
"in": "query",
"required": true
@@ -2320,7 +2320,7 @@
"BearerAuth": []
}
],
- "description": "Copies one or more user resources (identified by ID) into flow-{id}-data/resources/.\nFiles already present in the flow are skipped unless force=true, in which case they are replaced.",
+ "description": "Copies one or more user resources (identified by ID) into flow-{id}-data/resources/.\nFiles already present in the flow are skipped unless force=true, in which case they are replaced.\nWhen the primary container for this flow is running, new or replaced files are pushed to /work/resources/ (best-effort), same as uploads.",
"consumes": [
"application/json"
],
@@ -2409,7 +2409,6 @@
"BearerAuth": []
}
],
- "description": "Reads a file from flow-{id}-data/{sourcePath}, computes MD5, stores the blob, inserts a user_resources row.",
"consumes": [
"application/json"
],
@@ -2419,7 +2418,7 @@
"tags": [
"FlowFiles"
],
- "summary": "Promote a flow file to user resources",
+ "summary": "Promote a flow file or directory to user resources",
"parameters": [
{
"minimum": 0,
@@ -2441,7 +2440,7 @@
],
"responses": {
"200": {
- "description": "resource created or updated",
+ "description": "resources created or updated",
"schema": {
"allOf": [
{
@@ -2451,7 +2450,7 @@
"type": "object",
"properties": {
"data": {
- "$ref": "#/definitions/models.ResourceEntry"
+ "$ref": "#/definitions/models.ResourceList"
}
}
}
@@ -2471,7 +2470,7 @@
}
},
"404": {
- "description": "source file or flow not found",
+ "description": "source or flow not found",
"schema": {
"$ref": "#/definitions/ErrorResponse"
}
@@ -6886,7 +6885,7 @@
"type": "object",
"required": [
"destination",
- "sourcePath"
+ "source"
],
"properties": {
"destination": {
@@ -6897,8 +6896,8 @@
"description": "Force overwrites an existing resource at Destination if one already exists.",
"type": "boolean"
},
- "sourcePath": {
- "description": "SourcePath is a relative path within the flow cache, e.g. \"container/work/result.md\" or \"uploads/task.md\".",
+ "source": {
+ "description": "Source is a relative path within the flow cache, e.g. \"container/work/result.md\" or \"uploads/task.md\" or \"uploads/work/\".",
"type": "string"
}
}
@@ -6918,7 +6917,7 @@
"type": "array",
"minItems": 1,
"items": {
- "type": "string"
+ "type": "integer"
}
}
}
@@ -7243,10 +7242,10 @@
"id": {
"type": "string"
},
- "isDir": {
+ "is_dir": {
"type": "boolean"
},
- "modifiedAt": {
+ "modified_at": {
"type": "string"
},
"name": {
@@ -7518,10 +7517,10 @@
"id": {
"type": "string"
},
- "isDir": {
+ "is_dir": {
"type": "boolean"
},
- "modifiedAt": {
+ "modified_at": {
"type": "string"
},
"name": {
@@ -8061,13 +8060,13 @@
"models.ResourceEntry": {
"type": "object",
"properties": {
- "createdAt": {
+ "created_at": {
"type": "string"
},
"id": {
"type": "integer"
},
- "isDir": {
+ "is_dir": {
"type": "boolean"
},
"name": {
@@ -8079,10 +8078,10 @@
"size": {
"type": "integer"
},
- "updatedAt": {
+ "updated_at": {
"type": "string"
},
- "userId": {
+ "user_id": {
"type": "integer"
}
}
diff --git a/backend/pkg/server/docs/swagger.yaml b/backend/pkg/server/docs/swagger.yaml
index 7f396dba..4cbc737c 100644
--- a/backend/pkg/server/docs/swagger.yaml
+++ b/backend/pkg/server/docs/swagger.yaml
@@ -170,13 +170,13 @@ definitions:
description: Force overwrites an existing resource at Destination if one already
exists.
type: boolean
- sourcePath:
- description: SourcePath is a relative path within the flow cache, e.g. "container/work/result.md"
- or "uploads/task.md".
+ source:
+ description: Source is a relative path within the flow cache, e.g. "container/work/result.md"
+ or "uploads/task.md" or "uploads/work/".
type: string
required:
- destination
- - sourcePath
+ - source
type: object
models.AddResourcesRequest:
properties:
@@ -188,7 +188,7 @@ definitions:
description: IDs is the list of user resource IDs to copy into the flow resources
directory.
items:
- type: string
+ type: integer
minItems: 1
type: array
required:
@@ -424,9 +424,9 @@ definitions:
properties:
id:
type: string
- isDir:
+ is_dir:
type: boolean
- modifiedAt:
+ modified_at:
type: string
name:
type: string
@@ -615,9 +615,9 @@ definitions:
properties:
id:
type: string
- isDir:
+ is_dir:
type: boolean
- modifiedAt:
+ modified_at:
type: string
name:
type: string
@@ -994,11 +994,11 @@ definitions:
type: object
models.ResourceEntry:
properties:
- createdAt:
+ created_at:
type: string
id:
type: integer
- isDir:
+ is_dir:
type: boolean
name:
type: string
@@ -1006,9 +1006,9 @@ definitions:
type: string
size:
type: integer
- updatedAt:
+ updated_at:
type: string
- userId:
+ user_id:
type: integer
type: object
models.ResourceList:
@@ -2976,7 +2976,7 @@ paths:
name: flowID
required: true
type: integer
- - description: 'relative path in cache: uploads/<name> or container/<name>'
+ - description: 'relative path in cache: uploads/, resources/, or container/'
in: query
name: path
required: true
@@ -3162,7 +3162,7 @@ paths:
name: flowID
required: true
type: integer
- - description: 'relative path in cache: uploads/<name> or container/<name>'
+ - description: 'relative path in cache: uploads/, resources/, or container/'
in: query
name: path
required: true
@@ -3259,6 +3259,7 @@ paths:
description: |-
Copies one or more user resources (identified by ID) into flow-{id}-data/resources/.
Files already present in the flow are skipped unless force=true, in which case they are replaced.
+ When the primary container for this flow is running, new or replaced files are pushed to /work/resources/ (best-effort), same as uploads.
parameters:
- description: flow id
in: path
@@ -3313,8 +3314,6 @@ paths:
post:
consumes:
- application/json
- description: Reads a file from flow-{id}-data/{sourcePath}, computes MD5, stores
- the blob, inserts a user_resources row.
parameters:
- description: flow id
in: path
@@ -3332,13 +3331,13 @@ paths:
- application/json
responses:
"200":
- description: resource created or updated
+ description: resources created or updated
schema:
allOf:
- $ref: '#/definitions/SuccessResponse'
- properties:
data:
- $ref: '#/definitions/models.ResourceEntry'
+ $ref: '#/definitions/models.ResourceList'
type: object
"400":
description: invalid request
@@ -3349,7 +3348,7 @@ paths:
schema:
$ref: '#/definitions/ErrorResponse'
"404":
- description: source file or flow not found
+ description: source or flow not found
schema:
$ref: '#/definitions/ErrorResponse'
"409":
@@ -3362,7 +3361,7 @@ paths:
$ref: '#/definitions/ErrorResponse'
security:
- BearerAuth: []
- summary: Promote a flow file to user resources
+ summary: Promote a flow file or directory to user resources
tags:
- FlowFiles
/flows/{flowID}/graph:
diff --git a/backend/pkg/server/models/flow_files.go b/backend/pkg/server/models/flow_files.go
index e6082f82..8b9398ca 100644
--- a/backend/pkg/server/models/flow_files.go
+++ b/backend/pkg/server/models/flow_files.go
@@ -8,8 +8,8 @@ type FlowFile struct {
Name string `json:"name"`
Path string `json:"path"` // relative: "uploads/<x>" or resources/<x> or container/<x>"
Size int64 `json:"size"`
- IsDir bool `json:"isDir"`
- ModifiedAt time.Time `json:"modifiedAt"`
+ IsDir bool `json:"is_dir"`
+ ModifiedAt time.Time `json:"modified_at"`
}
// FlowFiles is the list response for flow file operations.
@@ -24,8 +24,8 @@ type ContainerFile struct {
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
- IsDir bool `json:"isDir"`
- ModifiedAt time.Time `json:"modifiedAt"`
+ IsDir bool `json:"is_dir"`
+ ModifiedAt time.Time `json:"modified_at"`
}
// ContainerFiles is the list response for container directory listing.
@@ -46,15 +46,15 @@ type PullFlowFilesRequest struct {
// AddResourcesRequest is the request body for copying user resources into a flow.
type AddResourcesRequest struct {
// IDs is the list of user resource IDs to copy into the flow resources directory.
- IDs []string `json:"ids" binding:"required,min=1"`
+ IDs []uint64 `json:"ids" binding:"required,min=1"`
// Force overwrites files that already exist in the flow resources directory.
Force bool `json:"force"`
}
// AddResourceFromFlowRequest is the request body for promoting a flow file to user resources.
type AddResourceFromFlowRequest struct {
- // SourcePath is a relative path within the flow cache, e.g. "container/work/result.md" or "uploads/task.md".
- SourcePath string `json:"sourcePath" binding:"required"`
+ // Source is a relative path within the flow cache, e.g. "container/work/result.md" or "uploads/task.md" or "uploads/work/".
+ Source string `json:"source" binding:"required"`
// Destination is the virtual path the resource will have in the user's resource tree.
Destination string `json:"destination" binding:"required"`
// Force overwrites an existing resource at Destination if one already exists.
diff --git a/backend/pkg/server/models/resources.go b/backend/pkg/server/models/resources.go
index 6861af63..dcf4af77 100644
--- a/backend/pkg/server/models/resources.go
+++ b/backend/pkg/server/models/resources.go
@@ -21,14 +21,14 @@ type UserResource struct {
// ResourceEntry is the REST response representation of a single resource.
type ResourceEntry struct {
- ID int64 `json:"id"`
- UserID uint64 `json:"userId"`
+ ID uint64 `json:"id"`
+ UserID uint64 `json:"user_id"`
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
- IsDir bool `json:"isDir"`
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
+ IsDir bool `json:"is_dir"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
}
// ResourceList is the REST list response for resource operations.
diff --git a/backend/pkg/server/services/flow_files.go b/backend/pkg/server/services/flow_files.go
index 934c3ad6..07f4528e 100644
--- a/backend/pkg/server/services/flow_files.go
+++ b/backend/pkg/server/services/flow_files.go
@@ -253,6 +253,7 @@ func (s *FlowFileService) UploadFlowFiles(c *gin.Context) {
// Temporary files are complete — move them into place, then push to container.
savedFiles := make([]models.FlowFile, 0, len(fileHeaders))
+ pushEntries := make([]flowfiles.TarEntry, 0, len(pending))
for i := range pending {
p := &pending[i]
if err := os.Rename(p.tmpPath, p.dstPath); err != nil {
@@ -276,18 +277,21 @@ func (s *FlowFileService) UploadFlowFiles(c *gin.Context) {
return
}
- // Best-effort push to the running primary container so the file is
- // immediately visible at /work/uploads/<name> without a container restart.
- if pushErr := s.copyFileToContainer(c.Request.Context(), flowID, p.dstPath); pushErr != nil {
- logger.FromContext(c).WithError(pushErr).WithFields(map[string]any{
- "flow_id": flowID,
- "file_name": p.fileName,
- }).Warn("uploaded file saved locally but could not be pushed to container")
- }
-
+ pushEntries = append(pushEntries, flowfiles.TarEntry{
+ LocalPath: p.dstPath,
+ TarPath: path.Join(flowfiles.UploadsDirName, p.fileName),
+ })
savedFiles = append(savedFiles, convertFlowFile(flowfiles.NewFile(info, flowfiles.UploadsDirName)))
}
+ // Best-effort: one tar stream + one CopyToContainer for all new uploads.
+ if pushErr := s.copyLocalFilesToPrimaryWork(c.Request.Context(), flowID, pushEntries); pushErr != nil {
+ logger.FromContext(c).WithError(pushErr).WithFields(map[string]any{
+ "flow_id": flowID,
+ "count": len(pushEntries),
+ }).Warn("uploaded files saved locally but could not be pushed to container")
+ }
+
sortFlowFiles(savedFiles)
s.publishFlowFilesAdded(c.Request.Context(), flow, savedFiles)
response.Success(c, http.StatusOK, models.FlowFiles{
@@ -302,7 +306,7 @@ func (s *FlowFileService) UploadFlowFiles(c *gin.Context) {
// @Produce json
// @Security BearerAuth
// @Param flowID path int true "flow id" minimum(0)
-// @Param path query string true "relative path in cache: uploads/<name> or container/<name>"
+// @Param path query string true "relative path in cache: uploads/, resources/, or container/"
// @Success 200 {object} response.successResp{data=models.FlowFiles} "flow file deleted successful"
// @Failure 400 {object} response.errorResp "invalid flow file request data"
// @Failure 403 {object} response.errorResp "deleting flow file not permitted"
@@ -343,7 +347,7 @@ func (s *FlowFileService) DeleteFlowFile(c *gin.Context) {
deletedFile := convertFlowFile(flowfiles.NewFileWithPath(info, filepath.ToSlash(c.Query("path"))))
if err := s.deleteUploadFromContainer(c.Request.Context(), flowID, c.Query("path")); err != nil {
- logger.FromContext(c).WithError(err).WithField("flow_id", flowID).Error("error deleting uploaded file from container")
+ logger.FromContext(c).WithError(err).WithField("flow_id", flowID).Error("error deleting file from primary container")
response.Error(c, response.ErrInternal, err)
return
}
@@ -367,7 +371,7 @@ func (s *FlowFileService) DeleteFlowFile(c *gin.Context) {
// @Produce octet-stream,application/zip,json
// @Security BearerAuth
// @Param flowID path int true "flow id" minimum(0)
-// @Param path query string true "relative path in cache: uploads/<name> or container/<name>"
+// @Param path query string true "relative path in cache: uploads/, resources/, or container/"
// @Success 200 {file} binary "file content, or ZIP archive for directories"
// @Failure 400 {object} response.errorResp "invalid flow file request data"
// @Failure 403 {object} response.errorResp "downloading flow file not permitted"
@@ -807,25 +811,25 @@ func (s *FlowFileService) resolveCachedPath(flowID uint64, reqPath string) (stri
return flowfiles.ResolveCachedPath(s.dataDir, flowID, reqPath)
}
-// copyFileToContainer packages a single local file as a TAR archive and uploads it
-// to the running primary container at /work/uploads/<filename>. Best-effort: errors are
-// returned but callers should only log them as warnings.
-func (s *FlowFileService) copyFileToContainer(ctx context.Context, flowID uint64, localFilePath string) error {
- if s.dockerClient == nil {
+// copyLocalFilesToPrimaryWork streams local regular files into the running primary
+// container under /work using TarPath for each entry (e.g. "uploads/a.txt",
+// "resources/dir/b.yml"). One tar archive and one CopyToContainer call per invocation.
+// Returns nil when Docker is unavailable, the container is not running, or entries is empty.
+func (s *FlowFileService) copyLocalFilesToPrimaryWork(ctx context.Context, flowID uint64, entries []flowfiles.TarEntry) error {
+ if s.dockerClient == nil || len(entries) == 0 {
return nil
}
containerName := primaryContainerName(flowID)
running, err := s.dockerClient.IsContainerRunning(ctx, containerName)
if err != nil || !running {
- return nil // container absent or not running — skip silently
+ return nil
}
pr, pw := io.Pipe()
errCh := make(chan error, 1)
go func() {
- tarName := path.Join(flowfiles.UploadsDirName, filepath.Base(localFilePath))
- errCh <- flowfiles.WriteSingleFileTar(pw, localFilePath, tarName)
+ errCh <- flowfiles.WriteFilesTar(pw, entries)
}()
copyErr := s.dockerClient.CopyToContainer(ctx, containerName, docker.WorkFolderPathInContainer, pr,
@@ -838,6 +842,42 @@ func (s *FlowFileService) copyFileToContainer(ctx context.Context, flowID uint64
return writeErr
}
+// pushResourcePathsToContainer copies flow resource files into the running primary
+// container at /work/resources/... in a single tar stream. Best-effort: skips when
+// Docker is unavailable or the container is not running.
+func (s *FlowFileService) pushResourcePathsToContainer(c *gin.Context, flowID uint64, relPaths []string) {
+ if len(relPaths) == 0 {
+ return
+ }
+
+ resourcesDir := flowfiles.FlowResourcesDir(s.dataDir, flowID)
+ prefixLen := len(flowfiles.ResourcesDirName) + 1
+ entries := make([]flowfiles.TarEntry, 0, len(relPaths))
+ for _, relPath := range relPaths {
+ if len(relPath) <= prefixLen {
+ continue
+ }
+ fsRelPath := relPath[prefixLen:]
+ entries = append(entries, flowfiles.TarEntry{
+ LocalPath: filepath.Join(resourcesDir, filepath.FromSlash(fsRelPath)),
+ TarPath: path.Join(flowfiles.ResourcesDirName, filepath.ToSlash(fsRelPath)),
+ })
+ }
+
+ if len(entries) == 0 {
+ return
+ }
+ if pushErr := s.copyLocalFilesToPrimaryWork(c.Request.Context(), flowID, entries); pushErr != nil {
+ logger.FromContext(c).WithError(pushErr).WithFields(map[string]any{
+ "flow_id": flowID,
+ "count": len(entries),
+ }).Warn("resource files saved locally but could not be pushed to container")
+ }
+}
+
+// deleteUploadFromContainer removes cache paths that mirror the primary container's
+// /work tree: uploads/... and resources/... Container mirror paths (container/...) stay
+// host-only and are not removed inside the running container.
func (s *FlowFileService) deleteUploadFromContainer(ctx context.Context, flowID uint64, reqPath string) error {
if s.dockerClient == nil {
return nil
@@ -845,7 +885,7 @@ func (s *FlowFileService) deleteUploadFromContainer(ctx context.Context, flowID
cleaned := filepath.Clean(filepath.FromSlash(strings.ReplaceAll(reqPath, "\\", "/")))
parts := strings.SplitN(cleaned, string(filepath.Separator), 2)
- if len(parts) == 0 || parts[0] != flowfiles.UploadsDirName {
+ if len(parts) == 0 || (parts[0] != flowfiles.UploadsDirName && parts[0] != flowfiles.ResourcesDirName) {
return nil
}
@@ -1035,6 +1075,7 @@ func primaryContainerName(flowID uint64) string {
// @Summary Copy user resources into a flow
// @Description Copies one or more user resources (identified by ID) into flow-{id}-data/resources/.
// @Description Files already present in the flow are skipped unless force=true, in which case they are replaced.
+// @Description When the primary container for this flow is running, new or replaced files are pushed to /work/resources/ (best-effort), same as uploads.
// @Tags FlowFiles
// @Accept json
// @Produce json
@@ -1093,16 +1134,16 @@ func (s *FlowFileService) AddResourcesToFlow(c *gin.Context) {
return
}
- found := make(map[string]models.UserResource, len(recs))
+ found := make(map[uint64]models.UserResource, len(recs))
for _, r := range recs {
- found[fmt.Sprintf("%d", r.ID)] = r
+ found[r.ID] = r
}
refs := make([]flowfiles.ResourceRef, 0, len(req.IDs))
- for _, idStr := range req.IDs {
- r, ok := found[idStr]
+ for _, id := range req.IDs {
+ r, ok := found[id]
if !ok {
- err := fmt.Errorf("resource %s not found", idStr)
+ err := fmt.Errorf("resource %d not found", id)
response.Error(c, response.ErrFlowFilesInvalidRequest, err)
return
}
@@ -1126,6 +1167,10 @@ func (s *FlowFileService) AddResourcesToFlow(c *gin.Context) {
return
}
+ if len(addedPaths) > 0 {
+ s.pushResourcePathsToContainer(c, flowID, addedPaths)
+ }
+
// Publish subscription events for newly added/updated files.
if len(addedPaths) > 0 {
addedFiles := make([]models.FlowFile, 0, len(addedPaths))
@@ -1173,19 +1218,21 @@ func (s *FlowFileService) AddResourcesToFlow(c *gin.Context) {
})
}
-// AddResourceFromFlow promotes a file from the flow cache into the user's global resource store.
-// @Summary Promote a flow file to user resources
-// @Description Reads a file from flow-{id}-data/{sourcePath}, computes MD5, stores the blob, inserts a user_resources row.
+// AddResourceFromFlow promotes a flow cache file or directory tree into the
+// user's global resource store. The response always lists every created or
+// updated entry, including any virtual parent directories that were ensured
+// during the promotion.
+// @Summary Promote a flow file or directory to user resources
// @Tags FlowFiles
// @Accept json
// @Produce json
// @Security BearerAuth
// @Param flowID path int true "flow id" minimum(0)
// @Param body body models.AddResourceFromFlowRequest true "source path, destination and force flag"
-// @Success 200 {object} response.successResp{data=models.ResourceEntry} "resource created or updated"
+// @Success 200 {object} response.successResp{data=models.ResourceList} "resources created or updated"
// @Failure 400 {object} response.errorResp "invalid request"
// @Failure 403 {object} response.errorResp "not permitted"
-// @Failure 404 {object} response.errorResp "source file or flow not found"
+// @Failure 404 {object} response.errorResp "source or flow not found"
// @Failure 409 {object} response.errorResp "destination already exists and force=false"
// @Failure 500 {object} response.errorResp "internal error"
// @Router /flows/{flowID}/files/to-resources [post]
@@ -1199,9 +1246,7 @@ func (s *FlowFileService) AddResourceFromFlow(c *gin.Context) {
uid := c.GetUint64("uid")
privs := c.GetStringSlice("prm")
- isAdmin := slices.Contains(privs, "resources.admin")
-
- if !isAdmin {
+ if !slices.Contains(privs, "resources.admin") {
if !slices.Contains(privs, "resources.upload") {
response.Error(c, response.ErrNotPermitted, nil)
return
@@ -1224,22 +1269,30 @@ func (s *FlowFileService) AddResourceFromFlow(c *gin.Context) {
return
}
- absPath, err := flowfiles.ResolveCachedPath(s.dataDir, flowID, req.SourcePath)
+ absPath, err := flowfiles.ResolveCachedPath(s.dataDir, flowID, req.Source)
if err != nil {
logger.FromContext(c).WithError(err).Error("invalid source path for add resource from flow")
response.Error(c, response.ErrFlowFilesInvalidData, err)
return
}
- if _, statErr := os.Lstat(absPath); statErr != nil {
+ info, statErr := os.Lstat(absPath)
+ if statErr != nil {
if os.IsNotExist(statErr) {
response.Error(c, response.ErrFlowFilesNotFound, statErr)
return
}
- logger.FromContext(c).WithError(statErr).Error("error stating flow file")
+ logger.FromContext(c).WithError(statErr).Error("error stating flow source")
response.Error(c, response.ErrInternal, statErr)
return
}
+ if !info.IsDir() && !info.Mode().IsRegular() {
+ // Reject symlinks and other special files: os.Open would follow
+ // symlinks at the OS level, bypassing the cache-root constraint.
+ response.Error(c, response.ErrFlowFilesInvalidData,
+ fmt.Errorf("source must be a regular file or a directory"))
+ return
+ }
destPath, err := resources.SanitizeResourcePath(req.Destination)
if err != nil {
@@ -1248,99 +1301,308 @@ func (s *FlowFileService) AddResourceFromFlow(c *gin.Context) {
return
}
- var existingRec models.UserResource
- existErr := s.db.Where("user_id = ? AND path = ?", uid, destPath).First(&existingRec).Error
- exists := existErr == nil
- if existErr != nil && !gorm.IsRecordNotFoundError(existErr) {
- logger.FromContext(c).WithError(existErr).Error("error checking resource existence")
- response.Error(c, response.ErrInternal, existErr)
- return
- }
- if exists && !req.Force {
- response.Error(c, response.ErrFlowFilesAlreadyExists, fmt.Errorf("resource already exists at %q; use force=true to overwrite", destPath))
+ saved, ok := s.promoteToResources(c, uid, absPath, destPath, info.IsDir(), req.Force)
+ if !ok {
return
}
+ response.Success(c, http.StatusOK,
+ models.ResourceList{Items: saved, Total: uint64(len(saved))})
+}
- f, err := os.Open(absPath)
- if err != nil {
- logger.FromContext(c).WithError(err).Error("error opening flow file")
+// promoteToResources promotes a single regular file or a directory tree from
+// the flow cache into the user's resource store. Mirrors the three-phase
+// pattern used by UploadResources:
+// 1. walk and stream every regular file to a temporary blob (MD5-keyed)
+// 2. atomically commit blobs into the content-addressed store
+// 3. persist all directory and file rows inside one DB transaction
+//
+// Returns the slice of resulting entries (newly-created dirs, added files,
+// updated files in that order) on success, or false after writing an error
+// response on failure.
+func (s *FlowFileService) promoteToResources(
+ c *gin.Context, uid uint64,
+ absSourceRoot, destPath string,
+ sourceIsDir, force bool,
+) ([]models.ResourceEntry, bool) {
+ log := logger.FromContext(c)
+ ctx := c.Request.Context()
+
+ if err := resources.EnsureResourcesDir(s.dataDir); err != nil {
+ log.WithError(err).Error("failed to ensure resources directory")
response.Error(c, response.ErrInternal, err)
- return
+ return nil, false
}
- tmpPath, hash, size, err := resources.SaveToTemp(f, resources.ResourcesDir(s.dataDir))
- f.Close()
- if err != nil {
- logger.FromContext(c).WithError(err).Error("error saving flow file to temp")
- response.Error(c, response.ErrInternal, err)
- return
+ blobsDir := resources.ResourcesDir(s.dataDir)
+
+ // ── Phase 1: stream every regular file to a temporary blob ────────────────
+ var pending []pendingResourceUpload
+ walkErr := filepath.Walk(absSourceRoot, func(filePath string, fi os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ // filepath.Walk uses Lstat → symlinks and special files are NOT regular
+ // and are skipped here (defense-in-depth against symlinked-target reads).
+ if !fi.Mode().IsRegular() {
+ return nil
+ }
+
+ var virtPath string
+ if sourceIsDir {
+ rel, relErr := filepath.Rel(absSourceRoot, filePath)
+ if relErr != nil {
+ return fmt.Errorf("relative path: %w", relErr)
+ }
+ sanitized, sanitizeErr := resources.SanitizeResourcePath(destPath + "/" + filepath.ToSlash(rel))
+ if sanitizeErr != nil {
+ log.WithError(sanitizeErr).Warnf("skipping unsafe file name %q", rel)
+ return nil
+ }
+ virtPath = sanitized
+ } else {
+ virtPath = destPath
+ }
+
+ // Phase 1 conflict pre-check: avoid IO when the destination obviously
+ // conflicts. Phase 3 re-checks inside the TX as the source of truth.
+ if !force {
+ _, exists, dbErr := findResourceByPath(s.db, uid, virtPath)
+ if dbErr != nil {
+ return fmt.Errorf("checking %s: %w", virtPath, dbErr)
+ }
+ if exists {
+ return fmt.Errorf("%w: resource already exists at %q", errResourceConflict, virtPath)
+ }
+ }
+
+ f, openErr := os.Open(filePath)
+ if openErr != nil {
+ return fmt.Errorf("open %s: %w", filePath, openErr)
+ }
+ tmpPath, hash, size, saveErr := resources.SaveToTemp(f, blobsDir)
+ f.Close()
+ if saveErr != nil {
+ return fmt.Errorf("save temp for %s: %w", filePath, saveErr)
+ }
+ pending = append(pending, pendingResourceUpload{
+ name: path.Base(virtPath),
+ vPath: virtPath,
+ tmpPath: tmpPath,
+ hash: hash,
+ size: size,
+ })
+ return nil
+ })
+ if walkErr != nil {
+ cleanupResourceUploads(pending)
+ if errors.Is(walkErr, errResourceConflict) {
+ response.Error(c, response.ErrFlowFilesAlreadyExists, walkErr)
+ } else {
+ log.WithError(walkErr).Error("error reading source files for promotion")
+ response.Error(c, response.ErrInternal, walkErr)
+ }
+ return nil, false
}
- if err := resources.CommitBlob(s.dataDir, hash, tmpPath); err != nil {
- logger.FromContext(c).WithError(err).Error("error committing blob for flow file")
- response.Error(c, response.ErrInternal, err)
- return
+ // ── Phase 2: commit blobs (atomic rename; idempotent on duplicate hashes) ─
+ for i := range pending {
+ p := &pending[i]
+ existed, blobErr := resources.BlobExists(s.dataDir, p.hash)
+ if blobErr != nil {
+ cleanupResourceUploads(pending[i:])
+ s.deleteOrphanBlobsIfUnreferenced(pending[:i])
+ log.WithError(blobErr).Error("error checking blob existence")
+ response.Error(c, response.ErrInternal, blobErr)
+ return nil, false
+ }
+ if commitErr := resources.CommitBlob(s.dataDir, p.hash, p.tmpPath); commitErr != nil {
+ cleanupResourceUploads(pending[i:])
+ s.deleteOrphanBlobsIfUnreferenced(pending[:i])
+ log.WithError(commitErr).Error("error committing blob")
+ response.Error(c, response.ErrInternal, commitErr)
+ return nil, false
+ }
+ p.tmpPath = ""
+ p.newBlob = !existed
}
- name := filepath.Base(destPath)
+ // Determine the directory chain to ensure: the destination itself for a
+ // directory promotion, or the parent of the file otherwise. May be empty
+ // when promoting a single file at the resource root.
+ parentDir := destPath
+ if !sourceIsDir {
+ if d := path.Dir(destPath); d != "." && d != "/" {
+ parentDir = d
+ } else {
+ parentDir = ""
+ }
+ }
- if exists {
- oldHash := existingRec.Hash
- if err := s.db.Model(&existingRec).Updates(models.UserResource{
- Hash: hash,
- Name: name,
- Size: size,
- UpdatedAt: time.Now(),
- }).Error; err != nil {
- logger.FromContext(c).WithError(err).Error("error updating resource record")
- response.Error(c, response.ErrInternal, err)
- return
+ // ── Phase 3: persist all rows in a single DB transaction ──────────────────
+ tx := s.db.Begin()
+ if tx.Error != nil {
+ s.deleteOrphanBlobsIfUnreferenced(pending)
+ log.WithError(tx.Error).Error("failed to begin transaction")
+ response.Error(c, response.ErrInternal, tx.Error)
+ return nil, false
+ }
+
+ var allDirs, added, updated []models.UserResource
+ var orphanHashes []string
+ var txErr error
+
+ if parentDir != "" {
+ rootDirs, _, rootOrphans, dirErr := ensureResourceDirs(tx, uid, parentDir, force)
+ if dirErr != nil {
+ txErr = dirErr
+ } else {
+ allDirs = append(allDirs, rootDirs...)
+ orphanHashes = append(orphanHashes, rootOrphans...)
}
- if s.db.Model(&models.UserResource{}).Where("hash = ?", oldHash).RecordNotFound() {
- os.Remove(resources.BlobPath(s.dataDir, oldHash))
+ }
+
+ // Ensure each unique parent directory chain only once across all files.
+ ensuredDirs := make(map[string]bool, len(pending))
+ if parentDir != "" {
+ ensuredDirs[parentDir] = true
+ }
+ for _, p := range pending {
+ if txErr != nil {
+ break
}
- entry := models.ResourceEntry{
- ID: int64(existingRec.ID),
- UserID: existingRec.UserID,
- Name: existingRec.Name,
- Path: existingRec.Path,
- Size: size,
- IsDir: false,
- CreatedAt: existingRec.CreatedAt,
- UpdatedAt: existingRec.UpdatedAt,
+
+ if subParent := path.Dir(p.vPath); subParent != "." && subParent != "/" && !ensuredDirs[subParent] {
+ ensuredDirs[subParent] = true
+ subDirs, _, subOrphans, dirErr := ensureResourceDirs(tx, uid, subParent, force)
+ if dirErr != nil {
+ txErr = dirErr
+ break
+ }
+ allDirs = append(allDirs, subDirs...)
+ orphanHashes = append(orphanHashes, subOrphans...)
}
- if s.ss != nil {
- s.ss.NewResourcePublisher(int64(uid)).ResourceUpdated(c.Request.Context(), convertResourceToModel(entry))
+
+ existing, exists, lookErr := findResourceByPath(tx, uid, p.vPath)
+ if lookErr != nil {
+ txErr = lookErr
+ break
}
- response.Success(c, http.StatusOK, entry)
- return
+ if exists {
+ if !force {
+ txErr = fmt.Errorf("%w: resource already exists at %q", errResourceConflict, p.vPath)
+ break
+ }
+ if existing.IsDir {
+ txErr = fmt.Errorf("%w: a directory already exists at %q", errResourceConflict, p.vPath)
+ break
+ }
+ if existing.Hash != "" && existing.Hash != p.hash {
+ orphanHashes = append(orphanHashes, existing.Hash)
+ }
+ now := time.Now()
+ if updErr := tx.Model(&existing).Updates(models.UserResource{
+ Hash: p.hash, Name: p.name, Size: p.size, UpdatedAt: now,
+ }).Error; updErr != nil {
+ txErr = updErr
+ break
+ }
+ existing.Hash, existing.Name, existing.Size, existing.UpdatedAt = p.hash, p.name, p.size, now
+ updated = append(updated, existing)
+ continue
+ }
+
+ rec := models.UserResource{
+ UserID: uid, Hash: p.hash, Name: p.name, Path: p.vPath, Size: p.size,
+ }
+ if createErr := tx.Create(&rec).Error; createErr != nil {
+ txErr = createErr
+ break
+ }
+ added = append(added, rec)
}
- rec := models.UserResource{
- UserID: uid,
- Hash: hash,
- Name: name,
- Path: destPath,
- Size: size,
- IsDir: false,
+ if txErr != nil {
+ tx.Rollback()
+ s.deleteOrphanBlobsIfUnreferenced(pending)
+ if isUniqueViolation(txErr) || errors.Is(txErr, errResourceConflict) {
+ response.Error(c, response.ErrFlowFilesAlreadyExists, txErr)
+ } else {
+ log.WithError(txErr).Error("failed to persist promoted resources")
+ response.Error(c, response.ErrInternal, txErr)
+ }
+ return nil, false
}
- if err := s.db.Create(&rec).Error; err != nil {
- logger.FromContext(c).WithError(err).Error("error inserting resource from flow")
- response.Error(c, response.ErrInternal, err)
- return
+ if commitErr := tx.Commit().Error; commitErr != nil {
+ tx.Rollback()
+ s.deleteOrphanBlobsIfUnreferenced(pending)
+ log.WithError(commitErr).Error("failed to commit promotion transaction")
+ response.Error(c, response.ErrInternal, commitErr)
+ return nil, false
}
- entry := models.ResourceEntry{
- ID: int64(rec.ID),
- UserID: rec.UserID,
- Name: rec.Name,
- Path: rec.Path,
- Size: rec.Size,
- IsDir: false,
- CreatedAt: rec.CreatedAt,
- UpdatedAt: rec.UpdatedAt,
- }
+ s.cleanupOrphanHashes(orphanHashes)
+
if s.ss != nil {
- s.ss.NewResourcePublisher(int64(uid)).ResourceAdded(c.Request.Context(), convertResourceToModel(entry))
+ pub := s.ss.NewResourcePublisher(int64(uid))
+ for _, r := range allDirs {
+ pub.ResourceAdded(ctx, convertResourceToModel(convertResource(r)))
+ }
+ for _, r := range added {
+ pub.ResourceAdded(ctx, convertResourceToModel(convertResource(r)))
+ }
+ for _, r := range updated {
+ pub.ResourceUpdated(ctx, convertResourceToModel(convertResource(r)))
+ }
+ }
+
+ out := convertResources(allDirs)
+ out = append(out, convertResources(added)...)
+ out = append(out, convertResources(updated)...)
+ return out, true
+}
+
+// deleteOrphanBlobsIfUnreferenced removes freshly-committed blobs from this
+// promotion ONLY if no DB row references them. Mirrors
+// ResourceService.deleteOrphanBlob — used after a rolled-back transaction or
+// a partial Phase 2 to avoid deleting blobs that other concurrent uploads of
+// identical content may now reference.
+func (s *FlowFileService) deleteOrphanBlobsIfUnreferenced(pending []pendingResourceUpload) {
+ for _, p := range pending {
+ if !p.newBlob || p.hash == "" {
+ continue
+ }
+ var count int64
+ if err := s.db.Model(&models.UserResource{}).
+ Where("hash = ?", p.hash).
+ Count(&count).Error; err != nil || count > 0 {
+ continue
+ }
+ _ = resources.DeleteBlob(s.dataDir, p.hash)
+ }
+}
+
+// cleanupOrphanHashes mirrors ResourceService.cleanupOrphanBlobs: it removes
+// .blob files whose hashes are no longer referenced by any user_resources row,
+// after a successful TX that may have orphaned blobs (overwrites).
+func (s *FlowFileService) cleanupOrphanHashes(hashes []string) {
+ if len(hashes) == 0 {
+ return
+ }
+ var stillReferenced []string
+ if err := s.db.Model(&models.UserResource{}).
+ Where("hash IN (?)", hashes).
+ Pluck("DISTINCT hash", &stillReferenced).Error; err != nil {
+ return
+ }
+ refSet := make(map[string]bool, len(stillReferenced))
+ for _, h := range stillReferenced {
+ refSet[h] = true
+ }
+ seen := make(map[string]bool, len(hashes))
+ for _, h := range hashes {
+ if h == "" || refSet[h] || seen[h] {
+ continue
+ }
+ seen[h] = true
+ _ = resources.DeleteBlob(s.dataDir, h)
}
- response.Success(c, http.StatusOK, entry)
}
diff --git a/backend/pkg/server/services/flow_files_test.go b/backend/pkg/server/services/flow_files_test.go
index 566623f1..60084738 100644
--- a/backend/pkg/server/services/flow_files_test.go
+++ b/backend/pkg/server/services/flow_files_test.go
@@ -769,18 +769,6 @@ func decodeContainerFilesResponse(t *testing.T, w *httptest.ResponseRecorder) mo
return resp.Data
}
-func decodeResourceEntryResponse(t *testing.T, w *httptest.ResponseRecorder) models.ResourceEntry {
- t.Helper()
-
- var resp struct {
- Status string `json:"status"`
- Data models.ResourceEntry `json:"data"`
- }
- require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp))
- require.Equal(t, "success", resp.Status)
- return resp.Data
-}
-
// flowFileMultipartBody builds a multipart body for upload tests.
type flowFileUploadFile struct {
name string
@@ -2376,7 +2364,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
privs: []string{"flow_files.upload", "resources.view"},
seedResources: []seedRes{{path: "creds/p.txt", content: "secret"}},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
},
wantStatus: http.StatusOK,
wantFileExists: "creds/p.txt",
@@ -2390,7 +2378,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
seedResources: []seedRes{{path: "creds/p.txt", content: "secret"}},
seedExisting: map[string]string{"creds/p.txt": "old"},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
},
wantStatus: http.StatusOK,
wantFileExists: "creds/p.txt",
@@ -2403,7 +2391,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
seedResources: []seedRes{{path: "creds/p.txt", content: "new"}},
seedExisting: map[string]string{"creds/p.txt": "old"},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
"force": true,
},
wantStatus: http.StatusOK,
@@ -2417,7 +2405,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
privs: []string{"flow_files.admin", "resources.admin"},
seedResources: []seedRes{{userID: 2, path: "creds/p.txt", content: "secret"}},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
},
wantStatus: http.StatusOK,
wantFileExists: "creds/p.txt",
@@ -2430,7 +2418,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
privs: []string{"flow_files.upload", "resources.view"},
seedResources: []seedRes{{userID: 2, path: "creds/p.txt", content: "secret"}},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
},
wantStatus: http.StatusForbidden,
},
@@ -2439,7 +2427,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"resources.view"},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
},
wantStatus: http.StatusForbidden,
},
@@ -2448,7 +2436,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"flow_files.upload"},
body: map[string]any{
- "ids": []string{"1"},
+ "ids": []uint64{1},
},
wantStatus: http.StatusForbidden,
},
@@ -2464,7 +2452,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"flow_files.upload", "resources.view"},
body: map[string]any{
- "ids": []string{},
+ "ids": []uint64{},
},
wantStatus: http.StatusBadRequest,
},
@@ -2473,7 +2461,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"flow_files.upload", "resources.view"},
body: map[string]any{
- "ids": []string{"999"},
+ "ids": []uint64{999},
},
wantStatus: http.StatusBadRequest,
},
@@ -2482,7 +2470,7 @@ func TestFlowFileService_AddResourcesToFlowScenarios(t *testing.T) {
uid: 1,
flowID: 99,
privs: []string{"flow_files.upload", "resources.view"},
- body: map[string]any{"ids": []string{"1"}},
+ body: map[string]any{"ids": []uint64{1}},
wantStatus: http.StatusNotFound,
},
}
@@ -2572,18 +2560,19 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
}
tests := []struct {
- name string
- flowOwner uint64
- uid uint64
- flowID uint64
- privs []string
- sourceFiles []sourceFile
- existingResource *models.UserResource
- body any
- rawBody string
- wantStatus int
- wantResourcePath string
- wantEventChannel string // "added" | "updated"
+ name string
+ flowOwner uint64
+ uid uint64
+ flowID uint64
+ privs []string
+ sourceFiles []sourceFile
+ sourceDirs []string // extra directories to create (without files)
+ existingResource *models.UserResource
+ body any
+ rawBody string
+ wantStatus int
+ wantResourcePaths []string // every path expected to be present in the ResourceList response and DB
+ wantEventChannel string // "added" | "updated"
}{
{
name: "promote uploaded file to new resource",
@@ -2591,12 +2580,12 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
privs: []string{"resources.upload", "flow_files.view"},
sourceFiles: []sourceFile{{relPath: "uploads/report.txt", content: "payload"}},
body: map[string]any{
- "sourcePath": "uploads/report.txt",
+ "source": "uploads/report.txt",
"destination": "promoted/report.txt",
},
- wantStatus: http.StatusOK,
- wantResourcePath: "promoted/report.txt",
- wantEventChannel: "added",
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{"promoted/report.txt"},
+ wantEventChannel: "added",
},
{
name: "promote container file to user resources",
@@ -2604,12 +2593,12 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
privs: []string{"resources.upload", "flow_files.view"},
sourceFiles: []sourceFile{{relPath: "container/etc/nginx.conf", content: "nginx"}},
body: map[string]any{
- "sourcePath": "container/etc/nginx.conf",
+ "source": "container/etc/nginx.conf",
"destination": "configs/nginx.conf",
},
- wantStatus: http.StatusOK,
- wantResourcePath: "configs/nginx.conf",
- wantEventChannel: "added",
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{"configs/nginx.conf"},
+ wantEventChannel: "added",
},
{
name: "force overwrite existing resource",
@@ -2620,13 +2609,13 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
Hash: md5HexForFlowFiles("old"), Name: "report.txt", Path: "promoted/report.txt", Size: 3,
},
body: map[string]any{
- "sourcePath": "uploads/report.txt",
+ "source": "uploads/report.txt",
"destination": "promoted/report.txt",
"force": true,
},
- wantStatus: http.StatusOK,
- wantResourcePath: "promoted/report.txt",
- wantEventChannel: "updated",
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{"promoted/report.txt"},
+ wantEventChannel: "updated",
},
{
name: "existing resource without force returns conflict",
@@ -2637,7 +2626,7 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
Hash: md5HexForFlowFiles("old"), Name: "report.txt", Path: "promoted/report.txt", Size: 3,
},
body: map[string]any{
- "sourcePath": "uploads/report.txt",
+ "source": "uploads/report.txt",
"destination": "promoted/report.txt",
},
wantStatus: http.StatusConflict,
@@ -2648,7 +2637,7 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
privs: []string{"resources.admin"},
sourceFiles: []sourceFile{{relPath: "uploads/report.txt", content: "payload"}},
body: map[string]any{
- "sourcePath": "uploads/report.txt",
+ "source": "uploads/report.txt",
"destination": "promoted/report.txt",
},
wantStatus: http.StatusForbidden,
@@ -2659,19 +2648,19 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
privs: []string{"resources.admin", "flow_files.admin"},
sourceFiles: []sourceFile{{relPath: "uploads/report.txt", content: "payload"}},
body: map[string]any{
- "sourcePath": "uploads/report.txt",
+ "source": "uploads/report.txt",
"destination": "promoted/report.txt",
},
- wantStatus: http.StatusOK,
- wantResourcePath: "promoted/report.txt",
- wantEventChannel: "added",
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{"promoted/report.txt"},
+ wantEventChannel: "added",
},
{
name: "missing resources.upload returns forbidden",
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"flow_files.view"},
body: map[string]any{
- "sourcePath": "uploads/report.txt", "destination": "promoted/report.txt",
+ "source": "uploads/report.txt", "destination": "promoted/report.txt",
},
wantStatus: http.StatusForbidden,
},
@@ -2680,7 +2669,7 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"resources.upload"},
body: map[string]any{
- "sourcePath": "uploads/report.txt", "destination": "promoted/report.txt",
+ "source": "uploads/report.txt", "destination": "promoted/report.txt",
},
wantStatus: http.StatusForbidden,
},
@@ -2689,7 +2678,7 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
uid: 1, flowID: 99,
privs: []string{"resources.upload", "flow_files.view"},
body: map[string]any{
- "sourcePath": "uploads/report.txt", "destination": "promoted/report.txt",
+ "source": "uploads/report.txt", "destination": "promoted/report.txt",
},
wantStatus: http.StatusNotFound,
},
@@ -2712,7 +2701,7 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"resources.upload", "flow_files.view"},
body: map[string]any{
- "sourcePath": "tmp/evil.txt", "destination": "promoted/report.txt",
+ "source": "tmp/evil.txt", "destination": "promoted/report.txt",
},
wantStatus: http.StatusBadRequest,
},
@@ -2721,7 +2710,7 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
flowOwner: 1, uid: 1, flowID: 1,
privs: []string{"resources.upload", "flow_files.view"},
body: map[string]any{
- "sourcePath": "uploads/missing.txt", "destination": "promoted/report.txt",
+ "source": "uploads/missing.txt", "destination": "promoted/report.txt",
},
wantStatus: http.StatusNotFound,
},
@@ -2731,10 +2720,125 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
privs: []string{"resources.upload", "flow_files.view"},
sourceFiles: []sourceFile{{relPath: "uploads/report.txt", content: "x"}},
body: map[string]any{
- "sourcePath": "uploads/report.txt", "destination": "../escape.txt",
+ "source": "uploads/report.txt", "destination": "../escape.txt",
},
wantStatus: http.StatusBadRequest,
},
+ // ── directory promotion ───────────────────────────────────────────────
+ {
+ name: "promote container directory with nested files to resources",
+ flowOwner: 1, uid: 1, flowID: 1,
+ privs: []string{"resources.upload", "flow_files.view"},
+ sourceFiles: []sourceFile{
+ {relPath: "container/scan/hosts.txt", content: "10.0.0.1"},
+ {relPath: "container/scan/sub/ports.txt", content: "80,443"},
+ },
+ body: map[string]any{
+ "source": "container/scan",
+ "destination": "promoted/scan",
+ },
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{
+ "promoted/scan",
+ "promoted/scan/hosts.txt",
+ "promoted/scan/sub",
+ "promoted/scan/sub/ports.txt",
+ },
+ wantEventChannel: "added",
+ },
+ {
+ name: "promote uploads directory to resources",
+ flowOwner: 1, uid: 1, flowID: 1,
+ privs: []string{"resources.upload", "flow_files.view"},
+ sourceFiles: []sourceFile{
+ {relPath: "uploads/payloads/shell.sh", content: "#!/bin/sh"},
+ {relPath: "uploads/payloads/enum.sh", content: "#!/bin/sh\nls"},
+ },
+ body: map[string]any{
+ "source": "uploads/payloads",
+ "destination": "scripts/payloads",
+ },
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{
+ "scripts/payloads",
+ "scripts/payloads/shell.sh",
+ "scripts/payloads/enum.sh",
+ },
+ wantEventChannel: "added",
+ },
+ {
+ name: "directory promotion without force returns conflict when file exists",
+ flowOwner: 1, uid: 1, flowID: 1,
+ privs: []string{"resources.upload", "flow_files.view"},
+ sourceFiles: []sourceFile{
+ {relPath: "uploads/data/report.txt", content: "new content"},
+ },
+ existingResource: &models.UserResource{
+ Hash: md5HexForFlowFiles("old content"), Name: "report.txt",
+ Path: "promoted/data/report.txt", Size: 11,
+ },
+ body: map[string]any{
+ "source": "uploads/data",
+ "destination": "promoted/data",
+ },
+ wantStatus: http.StatusConflict,
+ },
+ {
+ name: "directory promotion with force overwrites existing file",
+ flowOwner: 1, uid: 1, flowID: 1,
+ privs: []string{"resources.upload", "flow_files.view"},
+ sourceFiles: []sourceFile{
+ {relPath: "container/results/output.txt", content: "updated"},
+ },
+ existingResource: &models.UserResource{
+ Hash: md5HexForFlowFiles("original"), Name: "output.txt",
+ Path: "archive/results/output.txt", Size: 8,
+ },
+ body: map[string]any{
+ "source": "container/results",
+ "destination": "archive/results",
+ "force": true,
+ },
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{
+ "archive/results",
+ "archive/results/output.txt",
+ },
+ wantEventChannel: "updated",
+ },
+ {
+ name: "single file at deep path creates parent directory records",
+ flowOwner: 1, uid: 1, flowID: 1,
+ privs: []string{"resources.upload", "flow_files.view"},
+ sourceFiles: []sourceFile{{relPath: "uploads/note.txt", content: "deep"}},
+ body: map[string]any{
+ "source": "uploads/note.txt",
+ "destination": "deep/nested/folder/note.txt",
+ },
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{
+ "deep",
+ "deep/nested",
+ "deep/nested/folder",
+ "deep/nested/folder/note.txt",
+ },
+ wantEventChannel: "added",
+ },
+ {
+ name: "empty source directory creates only the destination dir entry",
+ flowOwner: 1, uid: 1, flowID: 1,
+ privs: []string{"resources.upload", "flow_files.view"},
+ sourceDirs: []string{"uploads/empty"},
+ body: map[string]any{
+ "source": "uploads/empty",
+ "destination": "promoted/empty",
+ },
+ wantStatus: http.StatusOK,
+ wantResourcePaths: []string{
+ "promoted/empty",
+ },
+ wantEventChannel: "added",
+ },
}
for _, tt := range tests {
@@ -2753,6 +2857,10 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
require.NoError(t, os.MkdirAll(filepath.Dir(abs), 0755))
require.NoError(t, os.WriteFile(abs, []byte(sf.content), 0644))
}
+ for _, sd := range tt.sourceDirs {
+ abs := filepath.Join(dataDir, fmt.Sprintf("flow-%d-data", tt.flowID), filepath.FromSlash(sd))
+ require.NoError(t, os.MkdirAll(abs, 0755))
+ }
if tt.existingResource != nil {
existing := *tt.existingResource
existing.UserID = tt.uid
@@ -2781,13 +2889,23 @@ func TestFlowFileService_AddResourceFromFlowScenarios(t *testing.T) {
if tt.wantStatus != http.StatusOK {
return
}
- entry := decodeResourceEntryResponse(t, w)
- assert.Equal(t, tt.wantResourcePath, entry.Path)
- assert.False(t, entry.IsDir)
- var rec models.UserResource
- require.NoError(t, db.Where("user_id = ? AND path = ?", tt.uid, tt.wantResourcePath).First(&rec).Error)
- assert.NotEmpty(t, rec.Hash)
+ list := decodeResourceListResponse(t, w)
+ // Response always carries at least every expected path; ancestor
+ // directory entries pushed by ensureResourceDirs may add extras.
+ assert.GreaterOrEqual(t, list.Total, uint64(len(tt.wantResourcePaths)))
+ itemsByPath := make(map[string]models.ResourceEntry, len(list.Items))
+ for _, item := range list.Items {
+ itemsByPath[item.Path] = item
+ }
+ for _, wantPath := range tt.wantResourcePaths {
+ _, inResponse := itemsByPath[wantPath]
+ assert.True(t, inResponse, "expected resource at path %q in response", wantPath)
+
+ var rec models.UserResource
+ require.NoError(t, db.Where("user_id = ? AND path = ?", tt.uid, wantPath).First(&rec).Error,
+ "resource record missing for path %q", wantPath)
+ }
if tt.wantEventChannel != "" {
events := ss.snapshot()
diff --git a/backend/pkg/server/services/resources.go b/backend/pkg/server/services/resources.go
index b5a89731..04be8f1f 100644
--- a/backend/pkg/server/services/resources.go
+++ b/backend/pkg/server/services/resources.go
@@ -1544,7 +1544,7 @@ func pathHasTrailingSeparator(p string) bool {
func convertResource(r models.UserResource) models.ResourceEntry {
return models.ResourceEntry{
- ID: int64(r.ID),
+ ID: r.ID,
UserID: r.UserID,
Name: r.Name,
Path: r.Path,
@@ -1565,7 +1565,7 @@ func convertResources(recs []models.UserResource) []models.ResourceEntry {
func convertResourceToModel(e models.ResourceEntry) *model.UserResource {
return &model.UserResource{
- ID: e.ID,
+ ID: int64(e.ID),
UserID: int64(e.UserID),
Name: e.Name,
Path: e.Path,
diff --git a/backend/pkg/server/services/resources_test.go b/backend/pkg/server/services/resources_test.go
index 81b5f986..04a55e13 100644
--- a/backend/pkg/server/services/resources_test.go
+++ b/backend/pkg/server/services/resources_test.go
@@ -2168,7 +2168,7 @@ func TestResourceService_ConvertResourceToModel(t *testing.T) {
modelResource := convertResourceToModel(entry)
require.NotNil(t, modelResource)
- assert.Equal(t, entry.ID, modelResource.ID)
+ assert.Equal(t, int64(entry.ID), modelResource.ID)
assert.Equal(t, int64(entry.UserID), modelResource.UserID)
assert.Equal(t, entry.Name, modelResource.Name)
assert.Equal(t, entry.Path, modelResource.Path)