Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(skills): store one catalog zip and cache compressed archives

wizardchen9e0333984f3d
diff --git a/internal/agent/skills/tenant_source.go b/internal/agent/skills/tenant_source.go
index 2aa3c683d..405bf15a9 100644
--- a/internal/agent/skills/tenant_source.go
+++ b/internal/agent/skills/tenant_source.go
@@ -21,14 +21,18 @@ import (
const (
maxBundleEntries = 20_000
maxBundleEntryBytes = 32 << 20 // 32 MiB per entry
- maxBundleBytes = 512 << 20 // 512 MiB across the archive
+ maxBundleBytes = 512 << 20 // 512 MiB across one archive (install cap)
)
-// cachedBundleCount bounds the unpacked archives one source keeps. A source
-// lives for a single agent run, and within that run read_skill is typically
-// called several times against the same skill, so a handful of entries removes
-// the repeated download without holding archives past the turn.
-const cachedBundleCount = 4
+// A source lives for a single agent run. read_skill is typically called
+// several times against the same skill, so a handful of compressed zips
+// removes the repeated download. 64 MiB is the keep-around budget; a zip
+// larger than that can stay as the sole occupant so a large skill does not
+// re-download on every file, without pinning several 100+ MiB archives.
+const (
+ cachedBundleCount = 4
+ cachedBundleBytes = 64 << 20
+)
// TenantSkillSource exposes the skills an administrator installed into one
// sandbox config's snapshot image.
@@ -44,27 +48,29 @@ type TenantSkillSource struct {
// between turns.
order []string
- // loadBundle fetches an uploaded archive by its stored reference. It may
- // be nil when the deployment cannot serve bundles, in which case only the
- // levels backed by the row are available.
- loadBundle func(ref string) ([]byte, error)
+ // loadBundle fetches the skill zip. It may be nil when the deployment
+ // cannot serve bundles, in which case only the levels backed by the row
+ // are available. Callers that leave install.BundleRef empty should resolve
+ // the catalog object here.
+ loadBundle func(row *types.TenantSkillEntity) ([]byte, error)
mu sync.Mutex
- // cache holds unpacked archives, most recently used first, keyed by
- // bundle_sha256: the archive is immutable under that key, so a hit can
- // never serve a stale tree.
+ // cache holds downloaded archives, most recently used first, keyed by
+ // bundle_sha256 (or the storage ref). The zip stays compressed: list and
+ // read_skill inflate one entry at a time so a 13k-file skill cannot pin
+ // hundreds of megabytes unpacked for the rest of the turn.
cache []cachedBundle
}
type cachedBundle struct {
- key string
- files map[string][]byte
+ key string
+ archive []byte
}
// NewTenantSkillSource builds a source over the rows of one sandbox config.
// Callers pass every row; the source itself decides which are usable.
func NewTenantSkillSource(
- rows []*types.TenantSkillEntity, loadBundle func(ref string) ([]byte, error),
+ rows []*types.TenantSkillEntity, loadBundle func(row *types.TenantSkillEntity) ([]byte, error),
) *TenantSkillSource {
src := &TenantSkillSource{
byName: make(map[string]*types.TenantSkillEntity, len(rows)),
@@ -154,14 +160,22 @@ func (s *TenantSkillSource) LoadSkillFile(name, relativePath string) (*SkillFile
if err != nil {
return nil, err
}
- files, err := s.bundleFiles(row)
+ archive, err := s.bundleArchive(row)
+ if err != nil {
+ return nil, err
+ }
+ index, err := skillBundleFileIndex(archive)
if err != nil {
return nil, err
}
- content, ok := files[clean]
+ item, ok := index[clean]
if !ok {
return nil, fmt.Errorf("file not found in skill %s: %s", name, relativePath)
}
+ content, err := readLimitedSkillZipFile(item)
+ if err != nil {
+ return nil, err
+ }
basePath, err := sandbox.SkillDirFor(row.Name)
if err != nil {
return nil, err
@@ -183,12 +197,16 @@ func (s *TenantSkillSource) ListSkillFiles(name string) ([]string, error) {
if err != nil {
return nil, err
}
- files, err := s.bundleFiles(row)
+ archive, err := s.bundleArchive(row)
if err != nil {
return nil, err
}
- names := make([]string, 0, len(files))
- for rel := range files {
+ index, err := skillBundleFileIndex(archive)
+ if err != nil {
+ return nil, err
+ }
+ names := make([]string, 0, len(index))
+ for rel := range index {
names = append(names, rel)
}
sort.Strings(names)
@@ -250,42 +268,42 @@ func safeSkillRelPath(relativePath string) (string, error) {
return clean, nil
}
-// bundleFiles returns the unpacked archive of one skill, downloading it at
+// bundleArchive returns the compressed zip of one skill, downloading it at
// most once per cache lifetime.
-func (s *TenantSkillSource) bundleFiles(
+func (s *TenantSkillSource) bundleArchive(
row *types.TenantSkillEntity,
-) (map[string][]byte, error) {
- ref := strings.TrimSpace(row.BundleRef)
- if ref == "" {
- return nil, fmt.Errorf(
- "skill %s has no stored bundle; its files cannot be read", row.Name)
- }
+) ([]byte, error) {
if s.loadBundle == nil {
return nil, fmt.Errorf("skill bundles are not available in this deployment")
}
key := strings.TrimSpace(row.BundleSHA256)
if key == "" {
- // Without a checksum the reference is the only stable key we have.
- key = ref
+ key = strings.TrimSpace(row.BundleRef)
+ }
+ if key == "" {
+ key = strings.TrimSpace(row.CatalogID)
+ }
+ if key == "" {
+ key = strings.TrimSpace(row.ID)
}
- if files := s.cached(key); files != nil {
- return files, nil
+ if archive := s.cached(key); archive != nil {
+ return archive, nil
}
- archive, err := s.loadBundle(ref)
+ archive, err := s.loadBundle(row)
if err != nil {
return nil, fmt.Errorf("download bundle of skill %s: %w", row.Name, err)
}
- files, err := unpackSkillBundle(archive)
- if err != nil {
- return nil, fmt.Errorf("read bundle of skill %s: %w", row.Name, err)
+ if len(archive) == 0 {
+ return nil, fmt.Errorf(
+ "skill %s has no stored bundle; its files cannot be read", row.Name)
}
- s.store(key, files)
- return files, nil
+ s.store(key, archive)
+ return archive, nil
}
-func (s *TenantSkillSource) cached(key string) map[string][]byte {
+func (s *TenantSkillSource) cached(key string) []byte {
s.mu.Lock()
defer s.mu.Unlock()
for i, entry := range s.cache {
@@ -294,25 +312,42 @@ func (s *TenantSkillSource) cached(key string) map[string][]byte {
}
s.cache = append(s.cache[:i], s.cache[i+1:]...)
s.cache = append([]cachedBundle{entry}, s.cache...)
- return entry.files
+ return entry.archive
}
return nil
}
-func (s *TenantSkillSource) store(key string, files map[string][]byte) {
+func (s *TenantSkillSource) store(key string, archive []byte) {
+ if key == "" || len(archive) == 0 || len(archive) > maxBundleBytes {
+ return
+ }
s.mu.Lock()
defer s.mu.Unlock()
- s.cache = append([]cachedBundle{{key: key, files: files}}, s.cache...)
- if len(s.cache) > cachedBundleCount {
- s.cache = s.cache[:cachedBundleCount]
+ for i, entry := range s.cache {
+ if entry.key != key {
+ continue
+ }
+ s.cache = append(s.cache[:i], s.cache[i+1:]...)
+ break
+ }
+ s.cache = append([]cachedBundle{{key: key, archive: archive}}, s.cache...)
+ for len(s.cache) > 1 &&
+ (len(s.cache) > cachedBundleCount || s.cachedBytes() > cachedBundleBytes) {
+ s.cache = s.cache[:len(s.cache)-1]
+ }
+}
+
+func (s *TenantSkillSource) cachedBytes() int {
+ total := 0
+ for _, entry := range s.cache {
+ total += len(entry.archive)
}
+ return total
}
-// unpackSkillBundle reads a skill archive into skill-root-relative paths. It
-// re-roots the archive at the directory holding SKILL.md, because an archive
-// wrapped in a single top-level directory is accepted at upload time and the
-// relative paths the model is given must match either shape.
-func unpackSkillBundle(archive []byte) (map[string][]byte, error) {
+// skillBundleFileIndex maps skill-root-relative paths to zip entries without
+// inflating file bodies.
+func skillBundleFileIndex(archive []byte) (map[string]*zip.File, error) {
reader, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
if err != nil {
return nil, fmt.Errorf("not a readable zip archive: %w", err)
@@ -321,8 +356,7 @@ func unpackSkillBundle(archive []byte) (map[string][]byte, error) {
return nil, fmt.Errorf("archive holds more than %d files", maxBundleEntries)
}
- raw := make(map[string][]byte, len(reader.File))
- var total int64
+ raw := make(map[string]*zip.File, len(reader.File))
for _, entry := range reader.File {
info := entry.FileInfo()
if info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
@@ -333,32 +367,31 @@ func unpackSkillBundle(archive []byte) (map[string][]byte, error) {
name == ".." || strings.HasPrefix(name, "../") {
return nil, fmt.Errorf("entry %q escapes the archive root", entry.Name)
}
- rc, err := entry.Open()
- if err != nil {
- return nil, fmt.Errorf("cannot read %q: %w", entry.Name, err)
- }
- content, err := io.ReadAll(io.LimitReader(rc, maxBundleEntryBytes+1))
- _ = rc.Close()
- if err != nil {
- return nil, fmt.Errorf("cannot read %q: %w", entry.Name, err)
- }
- if len(content) > maxBundleEntryBytes {
- return nil, fmt.Errorf("entry %q is too large", entry.Name)
- }
- total += int64(len(content))
- if total > maxBundleBytes {
- return nil, fmt.Errorf("archive is too large")
+ raw[name] = entry
+ }
+ prefix, err := skillZipRootPrefix(raw)
+ if err != nil {
+ return nil, err
+ }
+ if prefix == "" {
+ return raw, nil
+ }
+ out := make(map[string]*zip.File, len(raw))
+ for name, entry := range raw {
+ if !strings.HasPrefix(name, prefix+"/") {
+ continue
}
- raw[name] = content
+ out[strings.TrimPrefix(name, prefix+"/")] = entry
}
- return reRootSkillBundle(raw)
+ if _, ok := out[SkillFileName]; !ok {
+ return nil, fmt.Errorf("%s is missing from the archive", SkillFileName)
+ }
+ return out, nil
}
-// reRootSkillBundle strips a single wrapping directory when SKILL.md is not at
-// the archive root.
-func reRootSkillBundle(raw map[string][]byte) (map[string][]byte, error) {
+func skillZipRootPrefix(raw map[string]*zip.File) (string, error) {
if _, ok := raw[SkillFileName]; ok {
- return raw, nil
+ return "", nil
}
prefix := ""
for name := range raw {
@@ -370,19 +403,28 @@ func reRootSkillBundle(raw map[string][]byte) (map[string][]byte, error) {
continue
}
if prefix != "" && prefix != dir {
- return nil, fmt.Errorf("archive holds more than one skill")
+ return "", fmt.Errorf("archive holds more than one skill")
}
prefix = dir
}
if prefix == "" {
- return nil, fmt.Errorf("%s is missing from the archive", SkillFileName)
+ return "", fmt.Errorf("%s is missing from the archive", SkillFileName)
}
- out := make(map[string][]byte, len(raw))
- for name, content := range raw {
- if !strings.HasPrefix(name, prefix+"/") {
- continue
- }
- out[strings.TrimPrefix(name, prefix+"/")] = content
+ return prefix, nil
+}
+
+func readLimitedSkillZipFile(entry *zip.File) ([]byte, error) {
+ rc, err := entry.Open()
+ if err != nil {
+ return nil, fmt.Errorf("cannot read %q: %w", entry.Name, err)
}
- return out, nil
+ content, err := io.ReadAll(io.LimitReader(rc, maxBundleEntryBytes+1))
+ _ = rc.Close()
+ if err != nil {
+ return nil, fmt.Errorf("cannot read %q: %w", entry.Name, err)
+ }
+ if len(content) > maxBundleEntryBytes {
+ return nil, fmt.Errorf("entry %q is too large", entry.Name)
+ }
+ return content, nil
}
diff --git a/internal/agent/skills/tenant_source_test.go b/internal/agent/skills/tenant_source_test.go
index 05dc426ea..53bb2b9f9 100644
--- a/internal/agent/skills/tenant_source_test.go
+++ b/internal/agent/skills/tenant_source_test.go
@@ -69,7 +69,7 @@ func TestTenantSkillSourceBasePathIsTheImageDir(t *testing.T) {
func TestTenantSkillSourceHidesUnusableSkillsFromEveryLookup(t *testing.T) {
src := NewTenantSkillSource([]*types.TenantSkillEntity{
{ID: "sk-2", Name: "ready-disabled", Status: types.SkillStatusReady, Enabled: false},
- }, func(string) ([]byte, error) { return nil, errors.New("must not be called") })
+ }, func(*types.TenantSkillEntity) ([]byte, error) { return nil, errors.New("must not be called") })
_, err := src.GetSkillBasePath("ready-disabled")
require.Error(t, err)
@@ -90,7 +90,7 @@ func TestTenantSkillSourceLoadsInstructionsFromTheRow(t *testing.T) {
ID: "sk-1", Name: "pdf", Description: "PDF helpers",
Instructions: "Run scripts/extract.py.",
Status: types.SkillStatusReady, Enabled: true,
- }}, func(string) ([]byte, error) { return nil, errors.New("must not be called") })
+ }}, func(*types.TenantSkillEntity) ([]byte, error) { return nil, errors.New("must not be called") })
skill, err := src.LoadSkillInstructions("pdf")
@@ -113,8 +113,8 @@ func TestTenantSkillSourceReadsLevel3FilesFromTheBundle(t *testing.T) {
src := NewTenantSkillSource([]*types.TenantSkillEntity{{
ID: "sk-1", Name: "pdf", Status: types.SkillStatusReady, Enabled: true,
BundleRef: "local://sk-1.zip", BundleSHA256: "sha-1",
- }}, func(ref string) ([]byte, error) {
- require.Equal(t, "local://sk-1.zip", ref)
+ }}, func(row *types.TenantSkillEntity) ([]byte, error) {
+ require.Equal(t, "local://sk-1.zip", row.BundleRef)
return archive, nil
})
@@ -143,7 +143,7 @@ func TestTenantSkillSourceDownloadsEachBundleOnce(t *testing.T) {
src := NewTenantSkillSource([]*types.TenantSkillEntity{{
ID: "sk-1", Name: "pdf", Status: types.SkillStatusReady, Enabled: true,
BundleRef: "local://sk-1.zip", BundleSHA256: "sha-1",
- }}, func(string) ([]byte, error) {
+ }}, func(*types.TenantSkillEntity) ([]byte, error) {
downloads++
return archive, nil
})
@@ -161,7 +161,7 @@ func TestTenantSkillSourceRefusesPathsOutsideTheSkill(t *testing.T) {
src := NewTenantSkillSource([]*types.TenantSkillEntity{{
ID: "sk-1", Name: "pdf", Status: types.SkillStatusReady, Enabled: true,
BundleRef: "local://sk-1.zip",
- }}, func(string) ([]byte, error) {
+ }}, func(*types.TenantSkillEntity) ([]byte, error) {
return nil, errors.New("must not be called")
})
@@ -373,6 +373,16 @@ func preloadedSkillDir(t *testing.T, name, description string) string {
return root
}
+func TestTenantSkillSourceCacheKeepsOneOversizeArchive(t *testing.T) {
+ src := NewTenantSkillSource(nil, nil)
+ src.store("small", bytes.Repeat([]byte("s"), 32))
+ src.store("big", bytes.Repeat([]byte("b"), cachedBundleBytes+1))
+
+ require.Equal(t, bytes.Repeat([]byte("b"), cachedBundleBytes+1), src.cached("big"))
+ require.Nil(t, src.cached("small"),
+ "a zip over the keep-around budget must not sit next to other entries")
+}
+
func zipArchive(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
diff --git a/internal/application/service/agent_service.go b/internal/application/service/agent_service.go
index d05209c2f..03bb9637c 100644
--- a/internal/application/service/agent_service.go
+++ b/internal/application/service/agent_service.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
+ "strings"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/agent/approval"
@@ -569,11 +570,38 @@ func (s *agentService) tenantSkillSource(
// a caller ever creates the engine under a shorter-lived context, bundle
// downloads start failing for installed skills only, and loadBundle needs
// a ctx parameter.
- return skills.NewTenantSkillSource(rows, func(ref string) ([]byte, error) {
+ return skills.NewTenantSkillSource(rows, func(row *types.TenantSkillEntity) ([]byte, error) {
+ ref, err := s.skillBundleRef(ctx, ownerTenantID, row)
+ if err != nil {
+ return nil, err
+ }
return s.readSkillBundle(ctx, ownerTenantID, ref)
})
}
+func (s *agentService) skillBundleRef(
+ ctx context.Context, tenantID uint64, row *types.TenantSkillEntity,
+) (string, error) {
+ if row == nil {
+ return "", errors.New("skill is required")
+ }
+ if ref := strings.TrimSpace(row.BundleRef); ref != "" {
+ return ref, nil
+ }
+ cid := strings.TrimSpace(row.CatalogID)
+ if cid == "" || s.db == nil {
+ return "", fmt.Errorf("skill %s has no stored bundle; its files cannot be read", row.Name)
+ }
+ cat, err := repository.NewTenantSkillRepository(s.db).GetCatalog(ctx, tenantID, cid)
+ if err != nil {
+ return "", err
+ }
+ if cat == nil || strings.TrimSpace(cat.BundleRef) == "" {
+ return "", fmt.Errorf("skill %s has no stored bundle; its files cannot be read", row.Name)
+ }
+ return strings.TrimSpace(cat.BundleRef), nil
+}
+
// userEnvResolver builds the per-caller environment resolver for this run, or
// nil when there is no sandbox config to resolve against.
//
diff --git a/internal/application/service/tenant_skill_catalog.go b/internal/application/service/tenant_skill_catalog.go
index 57bd7278f..044c6c1b5 100644
--- a/internal/application/service/tenant_skill_catalog.go
+++ b/internal/application/service/tenant_skill_catalog.go
@@ -180,11 +180,11 @@ func (s *TenantSkillService) RegisterCatalogFromArchive(
func (s *TenantSkillService) RegisterCatalogFromSource(
ctx context.Context, tenantID uint64, source string,
) (*types.TenantSkillCatalogEntity, error) {
- archive, err := fetchSkillArchive(ctx, source, s.sourceHTTP)
+ bundle, archive, err := fetchNormalizedSkillBundle(ctx, source, s.sourceHTTP)
if err != nil {
return nil, err
}
- return s.RegisterCatalogFromArchive(ctx, tenantID, archive)
+ return s.upsertCatalogFromBundle(ctx, tenantID, bundle, archive, true)
}
// CatalogInstallResult is the per-sandbox outcome of one catalog install call.
@@ -337,6 +337,14 @@ func (s *TenantSkillService) storeCatalogBundle(
if catalog == nil || len(archive) == 0 {
return false, nil
}
+ digest := skillArchiveSHA256(archive)
+ if strings.TrimSpace(catalog.BundleRef) != "" &&
+ strings.TrimSpace(catalog.BundleSHA256) != "" &&
+ catalog.BundleSHA256 == digest {
+ // Same bytes already on the definition. Installing onto another
+ // sandbox must not mint a second object.
+ return true, nil
+ }
fs, err := s.fileServiceForTenant(ctx, tenantID)
if err != nil {
if requireStore {
@@ -345,6 +353,7 @@ func (s *TenantSkillService) storeCatalogBundle(
logger.Warnf(ctx, "[skill] catalog bundle storage unavailable: %v", err)
return false, nil
}
+ oldRef := strings.TrimSpace(catalog.BundleRef)
ref, err := fs.SaveBytes(ctx, archive, tenantID,
fmt.Sprintf("tenant-skills/catalog/%s.zip", catalog.ID), false)
if err != nil {
@@ -355,6 +364,9 @@ func (s *TenantSkillService) storeCatalogBundle(
return false, nil
}
catalog.BundleRef = ref
+ if oldRef != "" && oldRef != ref {
+ s.deleteBundleBestEffort(ctx, tenantID, oldRef)
+ }
return true, nil
}
diff --git a/internal/application/service/tenant_skill_catalog_test.go b/internal/application/service/tenant_skill_catalog_test.go
index eba1ecef9..76e2fd774 100644
--- a/internal/application/service/tenant_skill_catalog_test.go
+++ b/internal/application/service/tenant_skill_catalog_test.go
@@ -132,13 +132,17 @@ func TestInstallSkillStoresTheArchiveOnTheCatalog(t *testing.T) {
cat, err := fx.skillRepo.GetCatalogByName(ctx, 7, "pdf-tools")
require.NoError(t, err)
require.NotNil(t, cat)
- require.Equal(t, "file://bundle.zip", cat.BundleRef)
+ require.NotEmpty(t, cat.BundleRef)
+ require.Equal(t, 1, fx.savedBundles)
skill, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", id)
require.NoError(t, err)
require.Equal(t, cat.ID, skill.CatalogID)
- require.Equal(t, cat.BundleRef, skill.BundleRef,
- "the install row locates the same catalog object; it must not own a sandbox-scoped copy")
+ require.Empty(t, skill.BundleRef,
+ "the install row must not copy the catalog object; uninstall must not be able to delete it")
+ files, err := fx.svc.ListSkillFiles(ctx, 7, "cfg-1", id)
+ require.NoError(t, err)
+ require.NotEmpty(t, files)
}
func TestRemovingLastSandboxInstallKeepsTheCatalogArchive(t *testing.T) {
@@ -170,7 +174,7 @@ func TestRemovingLastSandboxInstallKeepsTheCatalogArchive(t *testing.T) {
require.Nil(t, gone)
require.NoError(t, fx.svc.DeleteCatalog(ctx, 7, cat.ID))
- require.Equal(t, []string{"file://bundle.zip"}, fx.deletedBundles,
+ require.Equal(t, []string{cat.BundleRef}, fx.deletedBundles,
"only deleting the skill from the catalog drops the stored zip")
}
@@ -210,4 +214,30 @@ func TestInstallCatalogToConfigsReportsPartialFailure(t *testing.T) {
require.NotNil(t, result)
require.Contains(t, result.Installs, "cfg-1")
require.Equal(t, "sandbox config not found", result.Errors["missing"])
+ require.Equal(t, 1, fx.savedBundles,
+ "installing onto a sandbox must reuse the zip already stored on the catalog")
+}
+
+func TestRegisterCatalogReplacesThePreviousZip(t *testing.T) {
+ fx := newInstallFixture(t)
+ ctx := context.Background()
+ first := zipBundle(t, map[string]string{
+ "SKILL.md": validSkillMD,
+ "scripts/extract.py": "print('v1')\n",
+ })
+ cat, err := fx.svc.RegisterCatalogFromArchive(ctx, 7, first)
+ require.NoError(t, err)
+ firstRef := cat.BundleRef
+ require.Equal(t, 1, fx.savedBundles)
+
+ second := zipBundle(t, map[string]string{
+ "SKILL.md": validSkillMD,
+ "scripts/extract.py": "print('v2')\n",
+ })
+ cat, err = fx.svc.RegisterCatalogFromArchive(ctx, 7, second)
+ require.NoError(t, err)
+ require.Equal(t, 2, fx.savedBundles)
+ require.NotEqual(t, firstRef, cat.BundleRef)
+ require.Equal(t, []string{firstRef}, fx.deletedBundles,
+ "replacing the definition zip must drop the previous object")
}
diff --git a/internal/application/service/tenant_skill_files.go b/internal/application/service/tenant_skill_files.go
index 46d1015c6..9a8f74c89 100644
--- a/internal/application/service/tenant_skill_files.go
+++ b/internal/application/service/tenant_skill_files.go
@@ -30,10 +30,13 @@ const (
skillFileEncodingBinary = "binary"
// The settings drawer lists the tree then immediately opens SKILL.md, and
- // every later click is another read of the same zip. A handful of recent
- // archives covers that without holding a 512 MiB upload in RAM.
+ // every later click is another read of the same zip. Keep a modest
+ // process-wide budget: 512 MiB is the install/decompress cap, not RAM
+ // this cache is allowed to pin. A zip larger than the budget is still
+ // cached so a large skill's drawer does not re-download on every click,
+ // but it is the only occupant until something else is opened.
skillBundleArchiveCacheSlots = 8
- skillBundleArchiveCacheBytes = 64 << 20 // 64 MiB across cached zips
+ skillBundleArchiveCacheBytes = 64 << 20
)
// SkillFileEntry is one path in an installed skill's stored archive.
@@ -221,8 +224,10 @@ func (s *TenantSkillService) storeSkillBundle(key string, archive []byte) {
}
type skillBundleArchiveCache struct {
- mu sync.Mutex
- entries []cachedSkillArchive
+ mu sync.Mutex
+ entries []cachedSkillArchive
+ slots int
+ maxBytes int
}
type cachedSkillArchive struct {
@@ -231,7 +236,10 @@ type cachedSkillArchive struct {
}
func newSkillBundleArchiveCache() *skillBundleArchiveCache {
- return &skillBundleArchiveCache{}
+ return &skillBundleArchiveCache{
+ slots: skillBundleArchiveCacheSlots,
+ maxBytes: skillBundleArchiveCacheBytes,
+ }
}
func (c *skillBundleArchiveCache) get(key string) []byte {
@@ -252,10 +260,7 @@ func (c *skillBundleArchiveCache) get(key string) []byte {
}
func (c *skillBundleArchiveCache) put(key string, archive []byte) {
- if c == nil || key == "" {
- return
- }
- if len(archive) > skillBundleArchiveCacheBytes {
+ if c == nil || key == "" || len(archive) == 0 {
return
}
c.mu.Lock()
@@ -268,7 +273,14 @@ func (c *skillBundleArchiveCache) put(key string, archive []byte) {
break
}
c.entries = append([]cachedSkillArchive{{key: key, archive: archive}}, c.entries...)
- for len(c.entries) > skillBundleArchiveCacheSlots || c.cachedBytes() > skillBundleArchiveCacheBytes {
+ slots, maxBytes := c.slots, c.maxBytes
+ if slots <= 0 {
+ slots = skillBundleArchiveCacheSlots
+ }
+ if maxBytes <= 0 {
+ maxBytes = skillBundleArchiveCacheBytes
+ }
+ for len(c.entries) > 1 && (len(c.entries) > slots || c.cachedBytes() > maxBytes) {
c.entries = c.entries[:len(c.entries)-1]
}
}
diff --git a/internal/application/service/tenant_skill_files_test.go b/internal/application/service/tenant_skill_files_test.go
index 32af40d21..f78522d10 100644
--- a/internal/application/service/tenant_skill_files_test.go
+++ b/internal/application/service/tenant_skill_files_test.go
@@ -46,6 +46,31 @@ func TestListSkillFilesReportsMissingBundle(t *testing.T) {
require.Equal(t, 404, appErr.HTTPCode)
}
+func TestListSkillFilesUsesCatalogWhenInstallHasNoRef(t *testing.T) {
+ fx := newInstallFixture(t)
+ ctx := context.Background()
+ archive, err := zipSkillFiles(map[string][]byte{
+ "SKILL.md": []byte(validSkillMD),
+ "scripts/extract.py": []byte("print('hi')\n"),
+ })
+ require.NoError(t, err)
+ fx.storedBundles = map[string][]byte{"file://catalog.zip": archive}
+ require.NoError(t, fx.skillRepo.CreateCatalog(ctx, &types.TenantSkillCatalogEntity{
+ ID: "cat-1", TenantID: 7, Name: "pdf-tools",
+ BundleRef: "file://catalog.zip", BundleSHA256: skillArchiveSHA256(archive),
+ }))
+ skill, err := fx.skillRepo.GetSkill(ctx, 7, "cfg-1", "sk-1")
+ require.NoError(t, err)
+ skill.CatalogID = "cat-1"
+ skill.BundleRef = ""
+ skill.Status = types.SkillStatusReady
+ require.NoError(t, fx.skillRepo.UpdateSkill(ctx, skill))
+
+ files, err := fx.svc.ListSkillFiles(ctx, 7, "cfg-1", "sk-1")
+ require.NoError(t, err)
+ require.NotEmpty(t, files)
+}
+
func TestReadSkillFileReturnsTextContent(t *testing.T) {
fx := newInstallFixture(t)
fx.seedStoredSkillBundle(t)
@@ -58,6 +83,38 @@ func TestReadSkillFileReturnsTextContent(t *testing.T) {
require.False(t, file.Binary)
}
+func TestSkillBundleArchiveCacheKeepsOversizeAsSoleOccupant(t *testing.T) {
+ c := &skillBundleArchiveCache{slots: 4, maxBytes: 8}
+ c.put("small", []byte("abcd"))
+ c.put("big", []byte("0123456789"))
+
+ require.Equal(t, []byte("0123456789"), c.get("big"))
+ require.Nil(t, c.get("small"),
+ "a zip over the keep-around budget must not sit next to other entries")
+}
+
+func TestSkillBundleArchiveCacheEvictsToStayUnderBudget(t *testing.T) {
+ c := &skillBundleArchiveCache{slots: 4, maxBytes: 8}
+ c.put("a", []byte("aaaa"))
+ c.put("b", []byte("bbbb"))
+ c.put("c", []byte("cccc"))
+
+ require.Equal(t, []byte("cccc"), c.get("c"))
+ require.Equal(t, []byte("bbbb"), c.get("b"))
+ require.Nil(t, c.get("a"))
+}
+
+func TestSkillBundleArchiveCacheRespectsSlotCount(t *testing.T) {
+ c := &skillBundleArchiveCache{slots: 2, maxBytes: 100}
+ c.put("a", []byte("a"))
+ c.put("b", []byte("b"))
+ c.put("c", []byte("c"))
+
+ require.Nil(t, c.get("a"))
+ require.Equal(t, []byte("c"), c.get("c"))
+ require.Equal(t, []byte("b"), c.get("b"))
+}
+
func TestListAndReadSkillFilesDownloadTheBundleOnce(t *testing.T) {
fx := newInstallFixture(t)
fx.seedStoredSkillBundle(t)
diff --git a/internal/application/service/tenant_skill_install.go b/internal/application/service/tenant_skill_install.go
index ea702d910..3cf2574fc 100644
--- a/internal/application/service/tenant_skill_install.go
+++ b/internal/application/service/tenant_skill_install.go
@@ -75,6 +75,15 @@ func (s *TenantSkillService) InstallSkill(
if err != nil {
return "", err
}
+ return s.installParsedSkill(ctx, tenantID, configID, bundle, archive)
+}
+
+func (s *TenantSkillService) installParsedSkill(
+ ctx context.Context, tenantID uint64, configID string, bundle *SkillBundle, archive []byte,
+) (string, error) {
+ if bundle == nil {
+ return "", fmt.Errorf("skill bundle is required")
+ }
// Re-uploading a skill by the same name is an upgrade of that skill, not a
// second row: the unique (config, name) index would reject the insert, and
@@ -132,8 +141,7 @@ func (s *TenantSkillService) InstallSkill(
// The zip lives on the catalog, not on this sandbox: uninstalling from
// the last config must not take the definition's files with it. The
- // install row only copies the catalog ref so read_skill / retry can
- // find the same object.
+ // install row only stores CatalogID; readers follow that to the zip.
catalog, err := s.upsertCatalogFromBundle(ctx, tenantID, bundle, archive, true)
if err != nil {
failCtx, cancelFail := s.cleanupContext(ctx)
@@ -1142,9 +1150,9 @@ func (s *TenantSkillService) fileServiceForTenant(
return fs, nil
}
-// pointInstallAtCatalog copies the catalog's stored zip onto the install row.
-// The bytes themselves stay on the definition: this is only a locator so a
-// sandbox-scoped read can find the same object after the catalog id is set.
+// pointInstallAtCatalog attaches the sandbox row to the definition that owns
+// the zip. The install does not copy BundleRef: readers follow CatalogID, and
+// uninstalling this sandbox must not be able to delete the definition object.
func (s *TenantSkillService) pointInstallAtCatalog(
ctx context.Context, skill *types.TenantSkillEntity, catalog *types.TenantSkillCatalogEntity,
) error {
@@ -1157,7 +1165,7 @@ func (s *TenantSkillService) pointInstallAtCatalog(
return s.updateSkillFields(ctx, skill.TenantID, skill.SandboxConfigID, skill.ID,
func(e *types.TenantSkillEntity) {
e.CatalogID = catalog.ID
- e.BundleRef = catalog.BundleRef
+ e.BundleRef = ""
})
}
diff --git a/internal/application/service/tenant_skill_install_test.go b/internal/application/service/tenant_skill_install_test.go
index ba4414994..3c497d492 100644
--- a/internal/application/service/tenant_skill_install_test.go
+++ b/internal/application/service/tenant_skill_install_test.go
@@ -762,9 +762,10 @@ func TestInstallSkillSkipsWhenReadyWithTheSameArchive(t *testing.T) {
require.Empty(t, fx.sessionCalls, "the same bytes must not boot a billed sandbox")
require.NotContains(t, fx.events, "create-snapshot")
require.Nil(t, fx.configRepo.saved, "the image pointer must stay where it is")
- require.GreaterOrEqual(t, fx.savedBundles, 1,
- "a no-op re-upload must still refresh the stored archive for read_skill")
- require.Equal(t, "file://bundle.zip", skill.BundleRef)
+ require.Equal(t, 1, fx.savedBundles,
+ "a no-op re-upload must not mint a second catalog object")
+ require.Empty(t, skill.BundleRef,
+ "the install row must not own a zip; readers follow CatalogID")
require.NotEmpty(t, skill.CatalogID,
"a skip must still attach the install to the workspace catalog")
}
@@ -3322,7 +3323,9 @@ func (s installFileService) SaveBytes(_ context.Context, data []byte, _ uint64,
}
copied := make([]byte, len(data))
copy(copied, data)
- s.fx.storedBundles["file://bundle.zip"] = copied
+ ref := fmt.Sprintf("file://bundle-%d.zip", s.fx.savedBundles)
+ s.fx.storedBundles[ref] = copied
+ return ref, nil
}
return "file://bundle.zip", nil
}
diff --git a/internal/application/service/tenant_skill_source.go b/internal/application/service/tenant_skill_source.go
index ad1675f57..bde7da08c 100644
--- a/internal/application/service/tenant_skill_source.go
+++ b/internal/application/service/tenant_skill_source.go
@@ -90,11 +90,11 @@ func (s *TenantSkillService) InstallSkillFromSource(
return "", apperrors.NewNotFoundError("sandbox config not found")
}
- archive, err := fetchSkillArchive(ctx, source, s.sourceHTTP)
+ bundle, archive, err := fetchNormalizedSkillBundle(ctx, source, s.sourceHTTP)
if err != nil {
return "", err
}
- return s.InstallSkill(ctx, tenantID, configID, archive)
+ return s.installParsedSkill(ctx, tenantID, configID, bundle, archive)
}
func skillSourceHTTPClient(override *http.Client) *http.Client {
@@ -110,16 +110,23 @@ func skillSourceHTTPClient(override *http.Client) *http.Client {
}
func fetchSkillArchive(ctx context.Context, source string, client *http.Client) ([]byte, error) {
+ _, archive, err := fetchNormalizedSkillBundle(ctx, source, client)
+ return archive, err
+}
+
+func fetchNormalizedSkillBundle(
+ ctx context.Context, source string, client *http.Client,
+) (*SkillBundle, []byte, error) {
parsed, err := parseSkillSource(source)
if err != nil {
- return nil, err
+ return nil, nil, err
}
httpClient := skillSourceHTTPClient(client)
fetched, err := fetchSkillSourceBytes(ctx, httpClient, parsed, 0)
if err != nil {
- return nil, err
+ return nil, nil, err
}
- return normalizeFetchedSkillArchive(fetched.body, fetched.contentType, fetched.subdir)
+ return normalizeFetchedSkill(fetched.body, fetched.contentType, fetched.subdir)
}
// parseSkillSource maps one paste onto exactly one kind. It does not probe
@@ -659,15 +666,26 @@ func getSkillURL(
}
func normalizeFetchedSkillArchive(body []byte, contentType, subdir string) ([]byte, error) {
+ _, archive, err := normalizeFetchedSkill(body, contentType, subdir)
+ return archive, err
+}
+
+func normalizeFetchedSkill(body []byte, contentType, subdir string) (*SkillBundle, []byte, error) {
if looksLikeSkillMarkdown(body) {
files := map[string][]byte{"SKILL.md": body}
- if _, err := skillBundleFromFiles(body, files); err != nil {
- return nil, err
+ bundle, err := skillBundleFromFiles(body, files)
+ if err != nil {
+ return nil, nil, err
}
- return zipSkillFiles(files)
+ archive, err := zipSkillFiles(files)
+ if err != nil {
+ return nil, nil, err
+ }
+ bundle.SHA256 = skillArchiveSHA256(archive)
+ return bundle, archive, nil
}
if !isZipPayload(contentType, body) {
- return nil, fmt.Errorf("%w: remote did not return a zip skill bundle", ErrSkillSourceInvalid)
+ return nil, nil, fmt.Errorf("%w: remote did not return a zip skill bundle", ErrSkillSourceInvalid)
}
opts := SkillBundleParseOptions{
Subdir: subdir,
@@ -676,9 +694,14 @@ func normalizeFetchedSkillArchive(body []byte, contentType, subdir string) ([]by
}
bundle, err := ParseSkillBundleWithOptions(body, opts)
if err != nil {
- return nil, err
+ return nil, nil, err
+ }
+ archive, err := zipSkillFiles(bundle.Files)
+ if err != nil {
+ return nil, nil, err
}
- return zipSkillFiles(bundle.Files)
+ bundle.SHA256 = skillArchiveSHA256(archive)
+ return bundle, archive, nil
}
func isZipMagic(body []byte) bool {
diff --git a/internal/types/tenant_skill.go b/internal/types/tenant_skill.go
index 25af15d8a..b0cc9084a 100644
--- a/internal/types/tenant_skill.go
+++ b/internal/types/tenant_skill.go
@@ -64,7 +64,9 @@ type TenantSkillEntity struct {
// Instructions is the SKILL.md body (level 2 disclosure).
Instructions string `gorm:"type:text"`
- // BundleRef locates the uploaded archive in the tenant's file service.
+ // BundleRef is a leftover locator from when each install owned a zip.
+ // New installs leave this empty: the catalog row owns the archive, and
+ // readers follow CatalogID. Older rows may still name an object.
BundleRef string `gorm:"type:varchar(1024)"`
BundleSHA256 string `gorm:"type:varchar(64)"`
@@ -145,6 +147,8 @@ type TenantSkillCatalogEntity struct {
Version string `gorm:"type:varchar(64)"`
Description string `gorm:"type:text"`
Instructions string `gorm:"type:text"`
+ // BundleRef locates the one stored zip for this definition. Install rows
+ // do not own a copy: sandbox uninstall must not delete this object.
BundleRef string `gorm:"type:varchar(1024)"`
BundleSHA256 string `gorm:"type:varchar(64)"`
CreatedAt time.Time