Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
refactor(skills): improve skill bundle size management and nginx configuration
wizardchen3a06a3b9e9bd
diff --git a/.env.example b/.env.exampleindex 59e86b478..59c1d21a5 100644--- a/.env.example+++ b/.env.example@@ -187,7 +187,7 @@ LOCAL_STORAGE_BASE_DIR=/data/files# LOCAL_STORAGE_PATH_PREFIX=# 统一文件大小限制(MB,默认 50)。影响知识库/附件单文件上传、docreader gRPC 消息、浏览器知识库校验。属部署期配置:Go/Nginx/docreader/浏览器四层启动时读一次,运行中改不生效,改后须同步重启四层。# MAX_FILE_SIZE_MB=50-# 技能 zip 上传与 GitHub/ClawHub 来源下载上限(MB,默认 256)。与知识库上传分开:ppt-master 一类技能包会超过 50MB。未设置时不低于 MAX_FILE_SIZE_MB,且不超过 512。Nginx 请求体取两者较大值。改后须重启 app 与 frontend。+# 技能 zip 上传与 GitHub/ClawHub 来源下载上限(MB,默认 256)。与知识库上传分开:ppt-master 一类技能包会超过 50MB。未设置时不低于 MAX_FILE_SIZE_MB,且不超过 512。Nginx 只对技能上传的两条路由放宽到这个值,其余路由仍是 MAX_FILE_SIZE_MB。改后须重启 app 与 frontend。# MAX_SKILL_BUNDLE_SIZE_MB=256# ========== B4. 对象存储 provider(按 STORAGE_TYPE 选其一)==========diff --git a/frontend/Dockerfile b/frontend/Dockerfileindex 0f2dc0160..7ccf9a0f6 100644--- a/frontend/Dockerfile+++ b/frontend/Dockerfile@@ -12,6 +12,10 @@ COPY dist /usr/share/nginx/htmlCOPY nginx.conf /etc/nginx/templates/default.conf.template+# Copied outside templates/ on purpose: envsubst would eat $host and the other+# nginx variables in it.+COPY nginx-api-proxy.conf /etc/nginx/api-proxy.conf+COPY docker-entrypoint.sh /docker-entrypoint.shRUN chmod +x /docker-entrypoint.shdiff --git a/frontend/docker-entrypoint.sh b/frontend/docker-entrypoint.shindex e72286bce..8da608292 100644--- a/frontend/docker-entrypoint.sh+++ b/frontend/docker-entrypoint.sh@@ -15,11 +15,6 @@ fiif [ "$SKILL_MB" -gt 512 ] 2>/dev/null; thenSKILL_MB=512fi-if [ "$FILE_MB" -gt "$SKILL_MB" ] 2>/dev/null; then- NGINX_MB=$FILE_MB-else- NGINX_MB=$SKILL_MB-ficat > /usr/share/nginx/html/config.js << EOFwindow.__RUNTIME_CONFIG__ = {@@ -29,12 +24,17 @@ window.__RUNTIME_CONFIG__ = {};EOF-# 处理 nginx 配置-export MAX_FILE_SIZE=${NGINX_MB}M+# 处理 nginx 配置。+# 两个上限分开注入:全站保持知识库的 MAX_FILE_SIZE,只有技能包上传的两条路由+# 放宽到 MAX_SKILL_BUNDLE_SIZE。合成一个全站上限会让每个上传端点都能收到+# 技能包那么大的 body。+export MAX_FILE_SIZE=${FILE_MB}M+export MAX_SKILL_BUNDLE_SIZE=${SKILL_MB}Mexport APP_HOST=${APP_HOST:-app}export APP_PORT=${APP_PORT:-8080}export APP_SCHEME=${APP_SCHEME:-http}-envsubst '${MAX_FILE_SIZE} ${APP_HOST} ${APP_PORT} ${APP_SCHEME}' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf+envsubst '${MAX_FILE_SIZE} ${MAX_SKILL_BUNDLE_SIZE} ${APP_HOST} ${APP_PORT} ${APP_SCHEME}' \+ < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf# 启动 nginxexec nginx -g 'daemon off;'diff --git a/frontend/nginx-api-proxy.conf b/frontend/nginx-api-proxy.confnew file mode 100644index 000000000..0eaaedd01--- /dev/null+++ b/frontend/nginx-api-proxy.conf@@ -0,0 +1,30 @@+# Shared backend proxy settings for the /api/ locations.+#+# Two locations proxy to the backend — the general one and the skill-upload one+# that raises client_max_body_size — and they must behave identically in every+# other respect, so the settings live here instead of being copied. Only+# proxy_pass stays in each location, since a regex location cannot carry a URI+# in it.+#+# This file is copied verbatim, not run through envsubst, so $host and friends+# survive as nginx variables.++proxy_set_header Host $host;+proxy_set_header X-Real-IP $remote_addr;+proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;+proxy_set_header X-Forwarded-Proto $scheme;++# 连接和重试配置+proxy_connect_timeout 30s; # 连接超时时间+proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;+proxy_next_upstream_tries 3; # 重试次数+proxy_next_upstream_timeout 30s; # 重试超时时间++# SSE 相关配置+proxy_http_version 1.1; # 使用 HTTP/1.1+proxy_set_header Connection ""; # 禁用 Connection: close,保持连接打开+chunked_transfer_encoding off; # 关闭分块传输编码+proxy_buffering off; # 关闭缓冲+proxy_cache off; # 关闭缓存+proxy_read_timeout 3600s; # 增加读取超时时间+proxy_send_timeout 3600s; # 增加发送超时时间diff --git a/frontend/nginx.conf b/frontend/nginx.confindex 11c8fd2cb..ebf6b61b6 100644--- a/frontend/nginx.conf+++ b/frontend/nginx.conf@@ -112,29 +112,25 @@ server {proxy_set_header X-Forwarded-Proto $scheme;}+ # 技能包上传的两条路由,body 上限放宽到 MAX_SKILL_BUNDLE_SIZE。+ # 单独成一个 location 而不是抬高全站上限:client_max_body_size 是按+ # location 生效的,写在 server 级会让知识库上传等端点也能收到技能包大小的+ # body。这里放在 /api/ 之前,正则 location 优先于前缀 location 命中。+ # 匹配上的 GET(列表、文件内容)不受影响——body 上限对无 body 的请求无意义。+ location ~ ^/api/v1/(?:skills/catalog|sandbox-configs/[^/]+/skills)(?:/|$) {+ client_max_body_size ${MAX_SKILL_BUNDLE_SIZE};++ # 正则 location 的 proxy_pass 不能带 URI,原始路径原样透传,+ # 与 /api/ 那条 `proxy_pass .../api/` 的改写结果一致。+ proxy_pass ${APP_SCHEME}://${APP_HOST}:${APP_PORT};+ include /etc/nginx/api-proxy.conf;+ }+# API请求代理到后端服务# APP_SCHEME 默认 http,远程 HTTPS 后端可设为 httpslocation /api/ {proxy_pass ${APP_SCHEME}://${APP_HOST}:${APP_PORT}/api/;- proxy_set_header Host $host;- proxy_set_header X-Real-IP $remote_addr;- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;- proxy_set_header X-Forwarded-Proto $scheme;-- # 连接和重试配置- proxy_connect_timeout 30s; # 连接超时时间- proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;- proxy_next_upstream_tries 3; # 重试次数- proxy_next_upstream_timeout 30s; # 重试超时时间-- # SSE 相关配置- proxy_http_version 1.1; # 使用 HTTP/1.1- proxy_set_header Connection ""; # 禁用 Connection: close,保持连接打开- chunked_transfer_encoding off; # 关闭分块传输编码- proxy_buffering off; # 关闭缓冲- proxy_cache off; # 关闭缓存- proxy_read_timeout 3600s; # 增加读取超时时间- proxy_send_timeout 3600s; # 增加发送超时时间+ include /etc/nginx/api-proxy.conf;}# 错误页面diff --git a/internal/agent/skills/tenant_source.go b/internal/agent/skills/tenant_source.goindex 405bf15a9..d4ab9aa14 100644--- a/internal/agent/skills/tenant_source.go+++ b/internal/agent/skills/tenant_source.go@@ -19,6 +19,16 @@ import (// archive read here is the same one that was accepted then, and a corrupted or// hostile object in storage must not be able to exhaust this process.const (+ // maxBundleZipEntries bounds central-directory iteration on the raw+ // archive, mirroring maxSkillBundleZipEntries. A GitHub zipball carries+ // the whole repository, so it sits well above the count of files kept as+ // the skill.+ maxBundleZipEntries = 100_000+ // maxBundleEntries caps the files kept as the skill, mirroring+ // maxSkillBundleFiles. It is counted after directory entries and the+ // zipball's extra trees are dropped, exactly as the install counts them:+ // applying it to the raw entry count instead would reject archives the+ // install accepted, taking read_skill down for a working install.maxBundleEntries = 20_000maxBundleEntryBytes = 32 << 20 // 32 MiB per entrymaxBundleBytes = 512 << 20 // 512 MiB across one archive (install cap)@@ -352,8 +362,8 @@ func skillBundleFileIndex(archive []byte) (map[string]*zip.File, error) {if err != nil {return nil, fmt.Errorf("not a readable zip archive: %w", err)}- if len(reader.File) > maxBundleEntries {- return nil, fmt.Errorf("archive holds more than %d files", maxBundleEntries)+ if len(reader.File) > maxBundleZipEntries {+ return nil, fmt.Errorf("archive holds more than %d entries", maxBundleZipEntries)}raw := make(map[string]*zip.File, len(reader.File))@@ -373,18 +383,21 @@ func skillBundleFileIndex(archive []byte) (map[string]*zip.File, error) {if err != nil {return nil, err}- if prefix == "" {- return raw, nil- }- out := make(map[string]*zip.File, len(raw))- for name, entry := range raw {- if !strings.HasPrefix(name, prefix+"/") {- continue+ out := raw+ if prefix != "" {+ out = make(map[string]*zip.File, len(raw))+ for name, entry := range raw {+ if !strings.HasPrefix(name, prefix+"/") {+ continue+ }+ out[strings.TrimPrefix(name, prefix+"/")] = entry+ }+ if _, ok := out[SkillFileName]; !ok {+ return nil, fmt.Errorf("%s is missing from the archive", SkillFileName)}- out[strings.TrimPrefix(name, prefix+"/")] = entry}- if _, ok := out[SkillFileName]; !ok {- return nil, fmt.Errorf("%s is missing from the archive", SkillFileName)+ if len(out) > maxBundleEntries {+ return nil, fmt.Errorf("archive holds more than %d files", maxBundleEntries)}return out, nil}diff --git a/internal/agent/skills/tenant_source_test.go b/internal/agent/skills/tenant_source_test.goindex 53bb2b9f9..24747bdb1 100644--- a/internal/agent/skills/tenant_source_test.go+++ b/internal/agent/skills/tenant_source_test.go@@ -5,6 +5,7 @@ import ("bytes""context""errors"+ "fmt""os""path/filepath""testing"@@ -383,6 +384,60 @@ func TestTenantSkillSourceCacheKeepsOneOversizeArchive(t *testing.T) {"a zip over the keep-around budget must not sit next to other entries")}+// The agent reads the same archive the install accepted, so the two have to+// count it the same way. Counting raw zip entries against the skill-file cap+// rejects bundles the install took — directory entries alone can carry a real+// skill past 20k — and read_skill then fails on an install that works.+const bundleIndexSkillMD = "---\nname: pdf\ndescription: d\n---\nbody\n"++func TestSkillBundleFileIndexCountsFilesTheWayTheInstallDid(t *testing.T) {+ files := map[string]string{"repo-main/" + SkillFileName: bundleIndexSkillMD}+ for i := 0; i < maxBundleEntries-1; i++ {+ files[fmt.Sprintf("repo-main/templates/asset-%d.txt", i)] = ""+ }+ // Directory entries push the raw count past the skill-file cap without+ // adding a single file the skill is made of.+ dirs := make([]string, 0, 2000)+ for i := 0; i < 2000; i++ {+ dirs = append(dirs, fmt.Sprintf("repo-main/templates/dir-%d/", i))+ }++ index, err := skillBundleFileIndex(zipArchiveWithDirs(t, files, dirs))++ require.NoError(t, err)+ require.Len(t, index, maxBundleEntries)+ require.Contains(t, index, SkillFileName)+}++func TestSkillBundleFileIndexRejectsMoreSkillFilesThanTheCap(t *testing.T) {+ files := map[string]string{"repo-main/" + SkillFileName: bundleIndexSkillMD}+ for i := 0; i < maxBundleEntries; i++ {+ files[fmt.Sprintf("repo-main/templates/asset-%d.txt", i)] = ""+ }++ _, err := skillBundleFileIndex(zipArchiveWithDirs(t, files, nil))++ require.ErrorContains(t, err, "more than 20000 files")+}++func zipArchiveWithDirs(t *testing.T, files map[string]string, dirs []string) []byte {+ t.Helper()+ var buf bytes.Buffer+ writer := zip.NewWriter(&buf)+ for _, name := range dirs {+ _, err := writer.Create(name)+ require.NoError(t, err)+ }+ for name, content := range files {+ entry, err := writer.Create(name)+ require.NoError(t, err)+ _, err = entry.Write([]byte(content))+ require.NoError(t, err)+ }+ require.NoError(t, writer.Close())+ return buf.Bytes()+}+func zipArchive(t *testing.T, files map[string]string) []byte {t.Helper()var buf bytes.Bufferdiff --git a/internal/application/service/tenant_sandbox_config.go b/internal/application/service/tenant_sandbox_config.goindex 9c8a95825..424c171ae 100644--- a/internal/application/service/tenant_sandbox_config.go+++ b/internal/application/service/tenant_sandbox_config.go@@ -221,6 +221,10 @@ type sandboxConfigSkillStore interface {MarkSnapshotState(ctx context.Context, tenantID uint64, id, state, snapshotID string) errorListSkillsByConfig(ctx context.Context, tenantID uint64, configID string) ([]*types.TenantSkillEntity, error)DeleteSkill(ctx context.Context, tenantID uint64, configID, skillID string) error+ // The tenant-wide lists answer who else still reads an archive the+ // deleted install rows had pinned.+ ListSkillsByTenant(ctx context.Context, tenantID uint64) ([]*types.TenantSkillEntity, error)+ ListCatalogsByTenant(ctx context.Context, tenantID uint64) ([]*types.TenantSkillCatalogEntity, error)DeleteSnapshotRowsByConfig(ctx context.Context, tenantID uint64, configID string) errorDeleteUserEnvVarsByConfig(ctx context.Context, tenantID uint64, configID string) error}@@ -1282,15 +1286,26 @@ func (s *TenantSandboxConfigService) cleanupSkillMetadata(if err != nil {logger.Warnf(ctx, "[sandbox] list skills for config %s cleanup failed: %v", configID, err)}+ var pinned []string+ seen := map[string]struct{}{}for _, skill := range skills {if skill == nil {continue}+ if ref := strings.TrimSpace(skill.BundleRef); ref != "" {+ if _, dup := seen[ref]; !dup {+ seen[ref] = struct{}{}+ pinned = append(pinned, ref)+ }+ }if err := s.skills.DeleteSkill(ctx, tenantID, configID, skill.ID); err != nil {logger.Warnf(ctx, "[sandbox] delete skill %s on config %s failed: %v",skill.ID, configID, err)}}+ // After the rows are gone, never before: each ref is named by the very row+ // being deleted, so asking first would always find a reader.+ s.releasePinnedBundles(ctx, tenantID, pinned)if err := s.skills.DeleteSnapshotRowsByConfig(ctx, tenantID, configID); err != nil {logger.Warnf(ctx, "[sandbox] delete snapshot ledger for config %s failed: %v",configID, err)@@ -1303,6 +1318,68 @@ func (s *TenantSandboxConfigService) cleanupSkillMetadata(}}+// releasePinnedBundles drops the archives whose last reader was an install row+// this config deletion just removed.+//+// An install row names an object only in one case: a re-register replaced the+// definition's copy while this sandbox kept running the image built from the+// old one, so the row pinned those bytes. Deleting the config drops that+// reader and no code path will ever look for them again.+//+// The pin is not exclusive, which is why this cannot just delete what the rows+// named. Sibling configs installed from the same archive pin the same object,+// and a definition may have been rolled back onto it, so the bytes go only once+// no catalog and no surviving install names them. A list that cannot be read+// keeps the archive: a leaked one costs storage, a deleted one costs some other+// sandbox its files.+func (s *TenantSandboxConfigService) releasePinnedBundles(+ ctx context.Context, tenantID uint64, refs []string,+) {+ if len(refs) == 0 || s.files == nil || s.skills == nil {+ return+ }+ catalogs, err := s.skills.ListCatalogsByTenant(ctx, tenantID)+ if err != nil {+ logger.Warnf(ctx, "[sandbox] list catalogs before releasing skill archives failed: %v", err)+ return+ }+ installs, err := s.skills.ListSkillsByTenant(ctx, tenantID)+ if err != nil {+ logger.Warnf(ctx, "[sandbox] list installs before releasing skill archives failed: %v", err)+ return+ }+ held := make(map[string]struct{}, len(catalogs)+len(installs))+ for _, cat := range catalogs {+ if cat != nil {+ held[strings.TrimSpace(cat.BundleRef)] = struct{}{}+ }+ }+ for _, row := range installs {+ if row != nil {+ held[strings.TrimSpace(row.BundleRef)] = struct{}{}+ }+ }++ var fs interfaces.FileService+ for _, ref := range refs {+ if _, still := held[ref]; still {+ continue+ }+ if fs == nil {+ resolved, _, resolveErr := s.files.ResolveFileService(ctx, &types.Tenant{ID: tenantID}, "", "", "")+ if resolveErr != nil || resolved == nil {+ logger.Warnf(ctx, "[sandbox] resolve file service to release skill archives failed: %v",+ resolveErr)+ return+ }+ fs = resolved+ }+ if err := fs.DeleteFile(ctx, ref); err != nil {+ logger.Warnf(ctx, "[sandbox] delete skill archive %s failed: %v", ref, err)+ }+ }+}+func (s *TenantSandboxConfigService) clientFor(cfg *types.TenantSandboxConfig,) (sandbox.ConfigSandboxClient, error) {diff --git a/internal/application/service/tenant_sandbox_config_test.go b/internal/application/service/tenant_sandbox_config_test.goindex e53eaf89c..461ac78c2 100644--- a/internal/application/service/tenant_sandbox_config_test.go+++ b/internal/application/service/tenant_sandbox_config_test.go@@ -1545,6 +1545,47 @@ func TestDeleteReleasesSkillSnapshotsBeforeSoftDelete(t *testing.T) {require.Equal(t, []string{"7:cfg-a"}, fx.skills.envVarsClearedFor)}+// An install row names an archive only when a re-register replaced the+// definition's copy and this sandbox kept running the image built from the old+// one. Deleting the config drops the only reader those bytes will ever have, so+// they have to go with it — nothing else records that the version existed.+func TestDeleteReclaimsArchivePinnedOnlyByThisConfig(t *testing.T) {+ fx := newSnapshotReleaseFixture(t, nil)+ fx.skills.skills[0].BundleRef = "bundle://skill-a-v1.zip"++ require.NoError(t, fx.svc.Delete(context.Background(), 7, "cfg-a", false))++ require.Equal(t, []string{"bundle://skill-a-v1.zip"}, fx.files.deleted)+}++func TestDeleteKeepsArchiveAnotherConfigStillPins(t *testing.T) {+ fx := newSnapshotReleaseFixture(t, nil)+ fx.skills.skills[0].BundleRef = "bundle://skill-a-v1.zip"+ // A sibling sandbox was built from the same replaced archive and is not+ // being deleted, so the pin is not this config's to release.+ fx.skills.skills = append(fx.skills.skills, &types.TenantSkillEntity{+ ID: "sk-2", TenantID: 7, SandboxConfigID: "cfg-b",+ BundleRef: "bundle://skill-a-v1.zip",+ })++ require.NoError(t, fx.svc.Delete(context.Background(), 7, "cfg-a", false))++ require.Empty(t, fx.files.deleted)+}++// A list that cannot be read is not evidence that nothing needs the bytes. A+// leaked archive costs storage; a deleted one costs some other sandbox its+// files, so an unreadable table keeps it.+func TestDeleteKeepsPinnedArchiveWhenReadersCannotBeListed(t *testing.T) {+ fx := newSnapshotReleaseFixture(t, nil)+ fx.skills.skills[0].BundleRef = "bundle://skill-a-v1.zip"+ fx.skills.listTenantErr = stderrors.New("database unavailable")++ require.NoError(t, fx.svc.Delete(context.Background(), 7, "cfg-a", false))++ require.Empty(t, fx.files.deleted)+}+func TestDeleteRefusesWhenSnapshotReleaseFailsWithoutForce(t *testing.T) {fx := newSnapshotReleaseFixture(t, map[string]error{"snap-2": stderrors.New("provider unavailable"),@@ -1700,6 +1741,12 @@ func newSnapshotReleaseFixture(t *testing.T, failDelete map[string]error) *snapsID: "sk-1", TenantID: 7, SandboxConfigID: "cfg-a",BundleRef: "bundle://skill-a.zip",}},+ // The install names the archive the definition itself holds, so+ // dropping the config must leave those bytes where they are.+ catalogs: []*types.TenantSkillCatalogEntity{{+ ID: "cat-1", TenantID: 7, Name: "skill-a",+ BundleRef: "bundle://skill-a.zip",+ }},},files: &deleteBundleResolver{},}@@ -1779,9 +1826,14 @@ func (c *snapshotReleaseClient) DeleteSnapshot(ctx context.Context, snapshotID stype deleteSkillStore struct {snapshots []*types.TenantSkillSnapshotEntityskills []*types.TenantSkillEntity+ catalogs []*types.TenantSkillCatalogEntitymarks []stringledgerCleared boolenvVarsClearedFor []string++ // listTenantErr makes the tenant-wide lookups fail, which is what decides+ // whether a pinned archive is kept rather than deleted on a bad read.+ listTenantErr error}func (s *deleteSkillStore) snapshot(id string) *types.TenantSkillSnapshotEntity {@@ -1846,6 +1898,44 @@ func (s *deleteSkillStore) ListSkillsByConfig(return out, nil}+func (s *deleteSkillStore) ListSkillsByTenant(+ ctx context.Context, tenantID uint64,+) ([]*types.TenantSkillEntity, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ if s.listTenantErr != nil {+ return nil, s.listTenantErr+ }+ var out []*types.TenantSkillEntity+ for _, row := range s.skills {+ if row.TenantID == tenantID {+ cp := *row+ out = append(out, &cp)+ }+ }+ return out, nil+}++func (s *deleteSkillStore) ListCatalogsByTenant(+ ctx context.Context, tenantID uint64,+) ([]*types.TenantSkillCatalogEntity, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ if s.listTenantErr != nil {+ return nil, s.listTenantErr+ }+ var out []*types.TenantSkillCatalogEntity+ for _, row := range s.catalogs {+ if row.TenantID == tenantID {+ cp := *row+ out = append(out, &cp)+ }+ }+ return out, nil+}+func (s *deleteSkillStore) DeleteSkill(ctx context.Context, tenantID uint64, configID, skillID string) error {if err := ctx.Err(); err != nil {return errdiff --git a/internal/application/service/tenant_skill_catalog.go b/internal/application/service/tenant_skill_catalog.goindex 4aea55dc7..857db4bc7 100644--- a/internal/application/service/tenant_skill_catalog.go+++ b/internal/application/service/tenant_skill_catalog.go@@ -284,7 +284,7 @@ func (s *TenantSkillService) upsertCatalogFromBundle(}now := s.now()if existing != nil {- stored, err := s.storeCatalogBundle(ctx, tenantID, existing, archive, requireStore)+ stored, releaseReplaced, err := s.storeCatalogBundle(ctx, tenantID, existing, archive, requireStore)if err != nil {return nil, err}@@ -297,8 +297,12 @@ func (s *TenantSkillService) upsertCatalogFromBundle(existing.BundleSHA256 = bundle.SHA256existing.UpdatedAt = nowif err := s.skills.UpdateCatalog(ctx, existing); err != nil {+ // The stored definition still names the old archive, so leave that+ // archive alone. The object just written is the leak, and it is the+ // harmless one: the next re-register overwrites that same key.return nil, err}+ releaseReplaced()return existing, nil}@@ -308,7 +312,7 @@ func (s *TenantSkillService) upsertCatalogFromBundle(Instructions: bundle.Instructions,CreatedAt: now, UpdatedAt: now,}- stored, err := s.storeCatalogBundle(ctx, tenantID, row, archive, requireStore)+ stored, releaseReplaced, err := s.storeCatalogBundle(ctx, tenantID, row, archive, requireStore)if err != nil {return nil, err}@@ -326,16 +330,33 @@ func (s *TenantSkillService) upsertCatalogFromBundle(if winner == nil {return nil, err}+ // The row that lost the race is never written, and the object above is+ // keyed by its ID, so nothing will ever name it again — the retry below+ // stores its own copy under the winner's key.+ if ref := strings.TrimSpace(row.BundleRef); ref != "" {+ s.deleteBundleBestEffort(ctx, tenantID, ref)+ }return s.upsertCatalogFromBundle(ctx, tenantID, bundle, archive, requireStore)}+ releaseReplaced()return row, nil}+// storeCatalogBundle writes the archive and reports whether the definition now+// names it, along with the work that settles the archive it replaced.+//+// That second return value has to be called by whoever commits the row, and+// only once the commit succeeds. Retiring the old object here instead would+// delete bytes the stored definition still points at the moment the write below+// fails: the row would name a ref that no longer resolves, and the object just+// saved would be unreferenced, which takes the definition's files down for good+// rather than leaving the failed re-register as a no-op.func (s *TenantSkillService) storeCatalogBundle(ctx context.Context, tenantID uint64, catalog *types.TenantSkillCatalogEntity, archive []byte, requireStore bool,-) (bool, error) {+) (bool, func(), error) {+ noop := func() {}if catalog == nil || len(archive) == 0 {- return false, nil+ return false, noop, nil}digest := skillArchiveSHA256(archive)if strings.TrimSpace(catalog.BundleRef) != "" &&@@ -343,15 +364,15 @@ func (s *TenantSkillService) storeCatalogBundle(catalog.BundleSHA256 == digest {// Same bytes already on the definition. Installing onto another// sandbox must not mint a second object.- return true, nil+ return true, noop, nil}fs, err := s.fileServiceForTenant(ctx, tenantID)if err != nil {if requireStore {- return false, err+ return false, noop, err}logger.Warnf(ctx, "[skill] catalog bundle storage unavailable: %v", err)- return false, nil+ return false, noop, nil}oldRef := strings.TrimSpace(catalog.BundleRef)oldSHA := strings.TrimSpace(catalog.BundleSHA256)@@ -359,16 +380,20 @@ func (s *TenantSkillService) storeCatalogBundle(fmt.Sprintf("tenant-skills/catalog/%s.zip", catalog.ID), false)if err != nil {if requireStore {- return false, fmt.Errorf("store catalog bundle: %w", err)+ return false, noop, fmt.Errorf("store catalog bundle: %w", err)}logger.Warnf(ctx, "[skill] catalog bundle store failed: %v", err)- return false, nil+ return false, noop, nil}catalog.BundleRef = ref- if oldRef != "" && oldRef != ref && !s.pinInstallsToReplacedBundle(ctx, tenantID, catalog, oldRef, oldSHA) {- s.deleteBundleBestEffort(ctx, tenantID, oldRef)+ if oldRef == "" || oldRef == ref {+ return true, noop, nil}- return true, nil+ return true, func() {+ if !s.pinInstallsToReplacedBundle(ctx, tenantID, catalog, oldRef, oldSHA) {+ s.deleteBundleBestEffort(ctx, tenantID, oldRef)+ }+ }, nil}// pinInstallsToReplacedBundle hands the archive a definition just replaced todiff --git a/internal/application/service/tenant_skill_catalog_test.go b/internal/application/service/tenant_skill_catalog_test.goindex 950c2b99c..95b3487f6 100644--- a/internal/application/service/tenant_skill_catalog_test.go+++ b/internal/application/service/tenant_skill_catalog_test.go@@ -242,6 +242,66 @@ func TestRegisterCatalogReplacesThePreviousZip(t *testing.T) {"replacing the definition zip must drop the previous object")}+// A re-register that cannot commit the row has to be a no-op. Retiring the+// previous object before the write would leave the stored definition naming a+// ref that no longer resolves, which takes its files down for good — far worse+// than the failed upload the caller asked about.+func TestRegisterCatalogKeepsThePreviousZipWhenTheRowFailsToCommit(t *testing.T) {+ fx := newInstallFixture(t)+ ctx := context.Background()+ first := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('v1')\n",+ })+ cat, err := fx.svc.RegisterCatalogFromArchive(ctx, 7, first)+ require.NoError(t, err)+ firstRef := cat.BundleRef++ fx.skillRepo.updateCatalogErr = errors.New("database unavailable")+ second := zipBundle(t, map[string]string{+ "SKILL.md": validSkillMD,+ "scripts/extract.py": "print('v2')\n",+ })+ _, err = fx.svc.RegisterCatalogFromArchive(ctx, 7, second)++ require.Error(t, err)+ require.Empty(t, fx.deletedBundles,+ "the stored definition still names the previous archive, so it must survive")++ stored, err := fx.skillRepo.GetCatalog(ctx, 7, cat.ID)+ require.NoError(t, err)+ require.Equal(t, firstRef, stored.BundleRef)+ require.NotEmpty(t, fx.storedBundles[firstRef],+ "the ref the row still names has to resolve to bytes")+}++// Two first-time registrations of one name race the unique index. The loser's+// object is keyed by the row ID that was never written, so nothing will ever+// name it again and the retry stores its own copy under the winner's key.+func TestRegisterCatalogDropsTheObjectOfTheRowThatLostTheNameRace(t *testing.T) {+ fx := newInstallFixture(t)+ ctx := context.Background()+ archive := zipBundle(t, map[string]string{"SKILL.md": validSkillMD})++ fx.skillRepo.createCatalogHook = func(e *types.TenantSkillCatalogEntity) error {+ // The winner appears between this caller's name lookup and its insert.+ fx.skillRepo.catalogs["cat-winner"] = &types.TenantSkillCatalogEntity{+ ID: "cat-winner", TenantID: e.TenantID, Name: e.Name,+ }+ fx.skillRepo.createCatalogHook = nil+ return errors.New("duplicate key value violates unique constraint")+ }++ cat, err := fx.svc.RegisterCatalogFromArchive(ctx, 7, archive)++ require.NoError(t, err)+ require.Equal(t, "cat-winner", cat.ID)+ require.Equal(t, 2, fx.savedBundles, "the retry stores the archive under the winner's key")+ require.Equal(t, []string{"file://bundle-1.zip"}, fx.deletedBundles,+ "the object of the row that was never written is unreachable and must go")+ require.NotEqual(t, "file://bundle-1.zip", cat.BundleRef)+}+// The sandbox that installed v1 goes on running v1 no matter what the// definition says next, so the archive it was built from has to outlive the// re-registration that supersedes it. Deleting it here is what would takediff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.goindex 3c497d492..0a03a38e4 100644--- a/internal/application/service/tenant_skill_install_test.go+++ b/internal/application/service/tenant_skill_install_test.go@@ -2258,8 +2258,14 @@ type installSkillRepo struct {listSnapshotsErr error// deleteSkillErr models the row delete failing past the point of no// return.- deleteSkillErr error- createErr error+ deleteSkillErr error+ createErr error+ // updateCatalogErr models the definition row failing to commit after its+ // new archive is already stored.+ updateCatalogErr error+ // createCatalogHook stands in for the insert so a test can have the unique+ // index reject this row because another request won the name first.+ createCatalogHook func(*types.TenantSkillCatalogEntity) errorgetByNameMisses intreadyWriteAttempts intdeleteSkillAttempts int@@ -2648,6 +2654,9 @@ func (r *installSkillRepo) CreateCatalog(_ context.Context, e *types.TenantSkillif r.catalogs == nil {r.catalogs = map[string]*types.TenantSkillCatalogEntity{}}+ if r.createCatalogHook != nil {+ return r.createCatalogHook(e)+ }cp := *er.catalogs[e.ID] = &cpreturn nil@@ -2692,6 +2701,9 @@ func (r *installSkillRepo) ListCatalogsByTenant(_ context.Context, tenantID uintfunc (r *installSkillRepo) UpdateCatalog(_ context.Context, e *types.TenantSkillCatalogEntity) error {r.mu.Lock()defer r.mu.Unlock()+ if r.updateCatalogErr != nil {+ return r.updateCatalogErr+ }if r.catalogs == nil {r.catalogs = map[string]*types.TenantSkillCatalogEntity{}}diff --git a/internal/handler/initialization.go b/internal/handler/initialization.goindex 82146f4a1..db67d65d8 100644--- a/internal/handler/initialization.go+++ b/internal/handler/initialization.go@@ -2221,9 +2221,23 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {return}+ // 验证文件大小 — MAX_FILE_SIZE_MB env (50MB 默认)。+ // 见 utils/filesize.go 注释:故意保留为部署期 env,不做 runtime setting。+ maxSizeMB := utils.GetMaxFileSizeMB()+ maxSize := maxSizeMB * 1024 * 1024+ // 上限必须在 multipart 解析之前生效:FormFile 会先把整个 body 缓冲下来+ // (超出内存部分落临时文件),之后的 header.Size 检查看到的已是收完的上传。+ // nginx 的 client_max_body_size 现在取的是更大的技能包上限,不再兜住这里。+ limitUploadBody(c, maxSize)+// 获取上传的图片文件file, header, err := c.Request.FormFile("image")if err != nil {+ if isRequestBodyTooLarge(err) {+ logger.Error(ctx, "File size too large")+ c.Error(errors.NewBadRequestError(fmt.Sprintf("图片文件大小不能超过%dMB", maxSizeMB)))+ return+ }logger.Error(ctx, "Failed to get uploaded image", err)c.Error(errors.NewBadRequestError("获取上传图片失败"))return@@ -2237,10 +2251,6 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {return}- // 验证文件大小 — MAX_FILE_SIZE_MB env (50MB 默认)。- // 见 utils/filesize.go 注释:故意保留为部署期 env,不做 runtime setting。- maxSizeMB := utils.GetMaxFileSizeMB()- maxSize := maxSizeMB * 1024 * 1024if header.Size > maxSize {logger.Error(ctx, "File size too large")c.Error(errors.NewBadRequestError(fmt.Sprintf("图片文件大小不能超过%dMB", maxSizeMB)))diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.goindex 7e4e4e8c0..67e16d26b 100644--- a/internal/handler/knowledge.go+++ b/internal/handler/knowledge.go@@ -331,20 +331,30 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {return}+ // Validate file size — read MAX_FILE_SIZE_MB env (50MB default).+ // Deliberately not a runtime system_setting; see filesize.go for the+ // rationale (nginx / docreader / browser bundle all cache this at+ // container startup, so a UI knob would silently mismatch).+ maxSizeMB := utils.GetMaxFileSizeMB()+ maxSize := maxSizeMB * 1024 * 1024+ // Capped before the multipart parse, not after: FormFile buffers the whole+ // body first, so the size check below only ever sees an upload we already+ // accepted. nginx no longer bounds this at the knowledge limit either — its+ // cap is the larger skill-bundle one.+ limitUploadBody(c, maxSize)+// Get the uploaded filefile, err := c.FormFile("file")if err != nil {+ if isRequestBodyTooLarge(err) {+ logger.Error(ctx, "File size too large")+ c.Error(errors.NewBadRequestError(fmt.Sprintf("文件大小不能超过%dMB", maxSizeMB)))+ return+ }logger.Error(ctx, "File upload failed", err)c.Error(errors.NewBadRequestError("File upload failed").WithDetails(err.Error()))return}-- // Validate file size — read MAX_FILE_SIZE_MB env (50MB default).- // Deliberately not a runtime system_setting; see filesize.go for the- // rationale (nginx / docreader / browser bundle all cache this at- // container startup, so a UI knob would silently mismatch).- maxSizeMB := utils.GetMaxFileSizeMB()- maxSize := maxSizeMB * 1024 * 1024if file.Size > maxSize {logger.Error(ctx, "File size too large")c.Error(errors.NewBadRequestError(fmt.Sprintf("文件大小不能超过%dMB", maxSizeMB)))diff --git a/internal/handler/model.go b/internal/handler/model.goindex 4f8233899..0aa297719 100644--- a/internal/handler/model.go+++ b/internal/handler/model.go@@ -363,6 +363,18 @@ func (h *ModelHandler) DebugModel(c *gin.Context) {return}+ // The optional file makes this a multipart body, so the cap has to be in+ // place before the first PostForm below — that call is what parses it, and+ // parsing is what buffers the upload to disk. Answer an oversized body here+ // too: the file is read with a `fileErr == nil` guard further down, which+ // would otherwise report a rejected upload as one that was never sent.+ limitUploadBody(c, secutils.GetMaxFileSize())+ if _, formErr := c.MultipartForm(); formErr != nil && isRequestBodyTooLarge(formErr) {+ c.Error(errors.NewBadRequestError(+ fmt.Sprintf("file cannot exceed %d MB", secutils.GetMaxFileSizeMB())))+ return+ }+input := c.PostForm("input")if len(input) > modelDebugMaxInputBytes {c.Error(errors.NewBadRequestError("input is too long"))diff --git a/internal/handler/sandbox_skill.go b/internal/handler/sandbox_skill.goindex a80cf663e..bda9b6ac5 100644--- a/internal/handler/sandbox_skill.go+++ b/internal/handler/sandbox_skill.go@@ -31,11 +31,6 @@ const (// and a connection that waits for that forever is a leak, so the stream// stops following and says so.skillEventMaxDuration = 60 * time.Minute-- // skillUploadEnvelopeSlack is the multipart framing allowed on top of the- // file itself, so the body cap refuses a genuinely oversized upload- // without rejecting a legal one for its boundary lines.- skillUploadEnvelopeSlack = 1 << 20)// sandboxSkillService is the skill surface the admin endpoints need. Reads and@@ -300,10 +295,7 @@ func (h *SandboxSkillHandler) GetFile(c *gin.Context) {// @Router /sandbox-configs/{id}/skills [post]func (h *SandboxSkillHandler) Upload(c *gin.Context) {maxBytes := secutils.GetMaxSkillBundleSize()- // The cap is applied to the body before it is parsed: multipart parsing- // buffers the whole request, spilling to temp files, so checking only the- // declared part size would let an unbounded body through first.- c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+skillUploadEnvelopeSlack)+ limitUploadBody(c, maxBytes)if strings.HasPrefix(c.ContentType(), "application/json") {h.installFromSource(c)@@ -312,8 +304,7 @@ func (h *SandboxSkillHandler) Upload(c *gin.Context) {file, header, err := c.Request.FormFile("file")if err != nil {- var tooLarge *http.MaxBytesError- if stderrors.As(err, &tooLarge) {+ if isRequestBodyTooLarge(err) {_ = c.Error(skillTooLargeError())return}diff --git a/internal/handler/skill_catalog.go b/internal/handler/skill_catalog.goindex 3aa058be2..5c91ecfc0 100644--- a/internal/handler/skill_catalog.go+++ b/internal/handler/skill_catalog.go@@ -51,7 +51,7 @@ func (h *SkillHandler) RegisterCatalog(c *gin.Context) {return}maxBytes := secutils.GetMaxSkillBundleSize()- c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+skillUploadEnvelopeSlack)+ limitUploadBody(c, maxBytes)if strings.HasPrefix(c.ContentType(), "application/json") {h.registerCatalogFromSource(c)@@ -60,8 +60,7 @@ func (h *SkillHandler) RegisterCatalog(c *gin.Context) {file, header, err := c.Request.FormFile("file")if err != nil {- var tooLarge *http.MaxBytesError- if stderrors.As(err, &tooLarge) {+ if isRequestBodyTooLarge(err) {_ = c.Error(skillTooLargeError())return}diff --git a/internal/handler/upload_limit.go b/internal/handler/upload_limit.gonew file mode 100644index 000000000..e8226e5b2--- /dev/null+++ b/internal/handler/upload_limit.go@@ -0,0 +1,37 @@+package handler++import (+ stderrors "errors"+ "net/http"++ "github.com/gin-gonic/gin"+)++// uploadEnvelopeSlack is the multipart framing allowed on top of the file+// itself, so a body cap refuses a genuinely oversized upload without+// rejecting a legal one for its boundary lines.+const uploadEnvelopeSlack = 1 << 20++// limitUploadBody caps the request body before anything parses it.+//+// It has to run before FormFile rather than after: ParseMultipartForm buffers+// the whole request, spilling to temp files, so a handler that only checks the+// declared part size has already accepted every byte by the time it looks. The+// frontend nginx used to be the only thing standing here, and its+// client_max_body_size is now the larger skill-bundle cap, so every+// knowledge-sized upload has to state its own limit.+func limitUploadBody(c *gin.Context, maxBytes int64) {+ c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes+uploadEnvelopeSlack)+}++// isRequestBodyTooLarge reports whether a multipart parsing error is the body+// cap firing rather than a malformed or absent part.+//+// The two need different answers. An oversized upload is a limit the caller+// can see and act on, while a missing field is a different request; a handler+// that treats the first as the second reports "no file was sent" for a file+// that was sent and rejected.+func isRequestBodyTooLarge(err error) bool {+ var tooLarge *http.MaxBytesError+ return stderrors.As(err, &tooLarge)+}diff --git a/internal/handler/upload_limit_test.go b/internal/handler/upload_limit_test.gonew file mode 100644index 000000000..ac5f75768--- /dev/null+++ b/internal/handler/upload_limit_test.go@@ -0,0 +1,101 @@+package handler++import (+ "bytes"+ "mime/multipart"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/gin-gonic/gin"+ "github.com/stretchr/testify/require"+)++// multipartBody frames a file of exactly size bytes the way a browser would,+// so the envelope slack is measured against real boundary lines.+func multipartBody(t *testing.T, size int) (string, []byte) {+ t.Helper()+ var buf bytes.Buffer+ writer := multipart.NewWriter(&buf)+ part, err := writer.CreateFormFile("file", "upload.bin")+ require.NoError(t, err)+ _, err = part.Write(bytes.Repeat([]byte("a"), size))+ require.NoError(t, err)+ require.NoError(t, writer.Close())+ return writer.FormDataContentType(), buf.Bytes()+}++func uploadLimitProbe(t *testing.T, maxBytes int64, fileSize int) (int, string) {+ t.Helper()+ gin.SetMode(gin.TestMode)+ engine := gin.New()+ engine.POST("/upload", func(c *gin.Context) {+ limitUploadBody(c, maxBytes)+ if _, err := c.FormFile("file"); err != nil {+ if isRequestBodyTooLarge(err) {+ c.String(http.StatusRequestEntityTooLarge, "too large")+ return+ }+ c.String(http.StatusBadRequest, err.Error())+ return+ }+ c.String(http.StatusOK, "ok")+ })++ contentType, body := multipartBody(t, fileSize)+ req := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(body))+ req.Header.Set("Content-Type", contentType)+ rec := httptest.NewRecorder()+ engine.ServeHTTP(rec, req)+ return rec.Code, rec.Body.String()+}++// The cap has to be the multipart parse's problem, not a size check that runs+// after it: a handler reaching FormFile has already let the body be buffered to+// a temp file, which is exactly the amplification the limit exists to stop.+func TestLimitUploadBodyRejectsAnOversizedUploadDuringTheParse(t *testing.T) {+ code, body := uploadLimitProbe(t, 1<<20, 4<<20)++ require.Equal(t, http.StatusRequestEntityTooLarge, code)+ require.Equal(t, "too large", body)+}++// The slack exists so the boundary lines of a legal upload cannot push it over.+func TestLimitUploadBodyAcceptsAFileAtExactlyTheCap(t *testing.T) {+ code, body := uploadLimitProbe(t, 1<<20, 1<<20)++ require.Equal(t, http.StatusOK, code)+ require.Equal(t, "ok", body)+}++// Every byte of framing still counts, so a body that clears the cap only by+// exceeding the slack is refused rather than silently allowed.+func TestLimitUploadBodyRejectsAFileBeyondTheSlack(t *testing.T) {+ code, _ := uploadLimitProbe(t, 1<<20, (1<<20)+uploadEnvelopeSlack+1)++ require.Equal(t, http.StatusRequestEntityTooLarge, code)+}++// A body cap firing and a request that named no file are different answers.+// Conflating them reports "file is required" for a file that was sent.+func TestIsRequestBodyTooLargeIgnoresAnAbsentPart(t *testing.T) {+ gin.SetMode(gin.TestMode)+ engine := gin.New()+ engine.POST("/upload", func(c *gin.Context) {+ limitUploadBody(c, 1<<20)+ _, err := c.FormFile("file")+ require.Error(t, err)+ require.False(t, isRequestBodyTooLarge(err))+ c.Status(http.StatusBadRequest)+ })++ var buf bytes.Buffer+ writer := multipart.NewWriter(&buf)+ require.NoError(t, writer.WriteField("input", strings.Repeat("a", 16)))+ require.NoError(t, writer.Close())++ req := httptest.NewRequest(http.MethodPost, "/upload", bytes.NewReader(buf.Bytes()))+ req.Header.Set("Content-Type", writer.FormDataContentType())+ engine.ServeHTTP(httptest.NewRecorder(), req)+}