Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

Sandbox skills mount file (git + registry) (#621)

Bhavesh Pateldb37b6e031c7
diff --git a/.changeset/20260907200000-sandbox-desired-skills.md b/.changeset/20260907200000-sandbox-desired-skills.md
new file mode 100644
index 000000000..ab07244ee
--- /dev/null
+++ b/.changeset/20260907200000-sandbox-desired-skills.md
@@ -0,0 +1,6 @@
+---
+"@truefoundry/trueforge": patch
+"@truefoundry/trueforge-core": patch
+---
+
+Sandbox skills: unify git and registry mounts onto `.tfy-desired-skills.json` (SkillMounter + skill_downloader), and always attach a mounter so existing skills are cleaned up.
diff --git a/packages/trueforge-core/scripts/generate-sandbox-scripts.mjs b/packages/trueforge-core/scripts/generate-sandbox-scripts.mjs
index a276ab440..a94f06e66 100644
--- a/packages/trueforge-core/scripts/generate-sandbox-scripts.mjs
+++ b/packages/trueforge-core/scripts/generate-sandbox-scripts.mjs
@@ -18,7 +18,7 @@ const read = f => readFileSync(join(scriptsDir, f), 'utf-8');
const out = `// AUTO-GENERATED by scripts/generate-sandbox-scripts.mjs — do not edit.
export const sandboxScripts = {
mcpClient: ${JSON.stringify(read('mcp_client.py'))},
- gitDownloader: ${JSON.stringify(read('git_downloader.py'))},
+ skillDownloader: ${JSON.stringify(read('skill_downloader.py'))},
} as const;
`;
writeFileSync(join(root, 'src/core/sandbox/sandboxScripts.gen.ts'), out);
diff --git a/packages/trueforge-core/src/core/index.ts b/packages/trueforge-core/src/core/index.ts
index 5ebbd311a..58394b7a4 100644
--- a/packages/trueforge-core/src/core/index.ts
+++ b/packages/trueforge-core/src/core/index.ts
@@ -176,4 +176,4 @@ export type { SandboxRefParts } from './sandbox/sandboxRef';
// Skills: the ISkillMounter seam lets hosts plug in their own skill sources
export { InstructionBuilder } from './InstructionBuilder';
export { SKILLS_PREAMBLE, SkillMounter, getSkillPath, renderSkillPromptBody } from './sandbox/skills';
-export type { GitSkill, ISkillMounter } from './sandbox/skills';
+export type { GitSkill, ISkillMounter, Skill } from './sandbox/skills';
diff --git a/packages/trueforge-core/src/core/sandbox/Sandbox.ts b/packages/trueforge-core/src/core/sandbox/Sandbox.ts
index 41740b8d3..4d7e79e3b 100644
--- a/packages/trueforge-core/src/core/sandbox/Sandbox.ts
+++ b/packages/trueforge-core/src/core/sandbox/Sandbox.ts
@@ -725,12 +725,19 @@ export class Sandbox extends LocalToolMCP {
}
// Fold skill installation into this single init exec. The mounter returns a declarative
- // command + env + timeout; runs even when empty so its downloader can prune a reused sandbox.
+ // command + env + timeout; runs even when empty so existing skills are cleaned up.
const skillInit = this.skillMounter?.getSandboxInit({
skillsDir,
- gitDownloaderPath: this.provider.getGitDownloaderPath(sandboxId),
+ skillDownloaderPath: this.provider.getSkillDownloaderPath(sandboxId),
});
if (skillInit) {
+ for (const upload of skillInit.uploads) {
+ await this.provider.uploadFile({
+ sandboxId,
+ remotePath: upload.remotePath,
+ content: upload.content,
+ });
+ }
initSteps.push(skillInit.command);
}
diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
index 1fba52e16..2332325c6 100644
--- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
+++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
@@ -499,7 +499,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
}
// Isolated container: image-absolute layout. GIT_CONFIG store --file needs an absolute path.
- // /opt/tf/{uploads,skills,tool-results,git_downloader.py,.git-credentials}
+ // /opt/tf/{uploads,skills,tool-results,skill_downloader.py,.git-credentials}
// /opt/tfy/mcp-client/mcp_client.py + /usr/local/bin/mcp-client (image PATH; no layout bin)
getToolResultDumpDir(): string {
return join('/opt', 'tf', 'tool-results');
@@ -517,7 +517,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
return join('/opt', 'tf', 'skills');
}
- getGitDownloaderPath(): string {
- return join('/opt', 'tf', 'git_downloader.py');
+ getSkillDownloaderPath(): string {
+ return join('/opt', 'tf', 'skill_downloader.py');
}
}
diff --git a/packages/trueforge-core/src/core/sandbox/provider/Provider.ts b/packages/trueforge-core/src/core/sandbox/provider/Provider.ts
index 6801c239d..0a46e5477 100644
--- a/packages/trueforge-core/src/core/sandbox/provider/Provider.ts
+++ b/packages/trueforge-core/src/core/sandbox/provider/Provider.ts
@@ -48,8 +48,24 @@ export interface SandboxExecParams {
timeoutSeconds?: number | undefined;
}
-// An init command a producer (e.g. a skill mounter) hands to the Sandbox, which supplies `sandboxId`.
-export type SandboxInit = Omit<SandboxExecParams, 'sandboxId'>;
+/** File uploaded before a sandbox init command runs. */
+export interface SandboxInitUpload {
+ remotePath: string;
+ content: Buffer;
+}
+
+/**
+ * Init step from a producer (e.g. skill mounter).
+ * Sandbox supplies `sandboxId` when it runs the command.
+ */
+export interface SandboxInit {
+ command: string;
+ env?: Record<string, string> | undefined;
+ /** Exec timeout for this init command (seconds). Skill downloads use a long value. */
+ timeoutSeconds: number;
+ /** Files to upload before the init command runs (e.g. requested skills JSON). */
+ uploads: readonly SandboxInitUpload[];
+}
export type SandboxBuildStatus = 'pending' | 'ready' | 'failed';
@@ -89,10 +105,10 @@ export interface SandboxProvider {
getGitCredentialsPath(sandboxId: string): string;
/** Directory for user-uploaded files (absolute, or cwd-relative when the provider has no global FS). */
getFileUploadsDir(sandboxId: string): string;
- /** Directory where git skills are materialized. */
+ /** Directory where skill mounts are materialized. */
getSkillsDir(sandboxId: string): string;
- /** Path the git skill downloader script is written to before it runs. */
- getGitDownloaderPath(sandboxId: string): string;
+ /** Path the skill downloader script is written to before it runs. */
+ getSkillDownloaderPath(sandboxId: string): string;
/** Downloads a file from the sandbox as a Buffer. Throws SandboxFileNotFoundError / SandboxNotAvailableError / SandboxPathIsDirectoryError / SandboxFileTooLargeError. */
downloadFile(params: { sandboxId: string; path: string }): Promise<Buffer>;
/** Uploads a file to the sandbox. */
diff --git a/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts
index 4dc6a05f2..03b6562ca 100644
--- a/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts
+++ b/packages/trueforge-core/src/core/sandbox/provider/TFYSandboxProvider.ts
@@ -258,7 +258,7 @@ export class TFYSandboxProvider implements SandboxProvider {
}
// Cwd-relative (no FS jail). exec() pwd-joins GIT_CONFIG / PATH / PYTHONPATH.
- // uploads, skills, tool-results, git_downloader.py, .git-credentials
+ // uploads, skills, tool-results, skill_downloader.py, .git-credentials
// mcp-client/mcp_client.py (no /usr/local/bin symlink)
getToolResultDumpDir(): string {
return 'tool-results';
@@ -276,7 +276,7 @@ export class TFYSandboxProvider implements SandboxProvider {
return 'skills';
}
- getGitDownloaderPath(): string {
- return 'git_downloader.py';
+ getSkillDownloaderPath(): string {
+ return 'skill_downloader.py';
}
}
diff --git a/packages/trueforge-core/src/core/sandbox/scripts/git_downloader.py b/packages/trueforge-core/src/core/sandbox/scripts/git_downloader.py
deleted file mode 100644
index 516c5d1ed..000000000
--- a/packages/trueforge-core/src/core/sandbox/scripts/git_downloader.py
+++ /dev/null
@@ -1,545 +0,0 @@
-#!/usr/bin/env python3
-
-# /// script
-# dependencies = ["pydantic==2.12.5"]
-# ///
-
-"""Materialize git skill directories into TFY_SKILLS_DIR (default /opt/tfy/skills). Requires `git`.
-
-Each git skill (AGENT_GIT_SKILLS) is materialized with a blob-filtered sparse `git` clone that
-fetches only the requested subdir at a resolved object id, keyed by name. Mount `ref` values
-(branch, tag, or full object id) are resolved here via `git ls-remote` — the host never spawns
-git. A single on-disk state file records the installed object id + subdir per name so a skill
-whose tip hasn't moved is skipped instead of re-cloning every run, and skills no longer desired
-are pruned from disk. A sparse clone is used instead of a full repo tarball because its cost is
-~constant in the subdir size rather than the whole-repo size, which is dramatically faster for a
-small skill living in a large monorepo.
-
-Optional env:
- - AGENT_GIT_SKILLS (base64-encoded JSON list of {name, url, path, ref}; empty
- clears git downloads). `ref` is a branch, tag, or full object id.
- - TFY_SKILLS_DIR: override the skills directory (default /opt/tfy/skills).
-"""
-
-from __future__ import annotations
-
-import base64
-import json
-import os
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
-
-DEFAULT_SKILLS_DIR = "/opt/tfy/skills"
-STATE_FILE_NAME = ".tfy-skill-downloader-state.json"
-# Per-git-invocation wall-clock cap so a hung/slow fetch can't stall sandbox init indefinitely.
-GIT_CLONE_TIMEOUT_SECONDS = 120
-# Cap the installed skill size so a huge repo-root skill can't fill the persistent skills dir. The
-# sparse clone already bounds a subdir skill to its subdir; this also guards the whole-repo (root
-# subdir) case where checkout hydrates every file.
-GIT_SKILL_MAX_BYTES = 200 * 1024 * 1024 # 200MB installed
-# Resolved object id used for fetch + skip state (sha1 / sha256).
-OBJECT_ID_RE = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$")
-# Mount ref (branch/tag/SHA) charset — must match host AgentSpec SkillMount.ref validation.
-# Validated before the value is passed to `git ls-remote` / used as a fetch arg fallback.
-GIT_REF_RE = re.compile(r"^[A-Za-z0-9._\-/]+$")
-SKILLS_ROOT = Path(os.environ.get("TFY_SKILLS_DIR", DEFAULT_SKILLS_DIR))
-STATE_PATH = SKILLS_ROOT / STATE_FILE_NAME
-
-
-class GitSkillError(Exception):
- """Recoverable per-repo/per-skill git failure. Raised (not sys.exit) so a single bad repo or
- subdirectory doesn't abort the remaining repository groups (best-effort failure isolation)."""
-
-
-def skill_dir(name: str) -> Path | None:
- """Resolve a skill's on-disk directory. This is the ONLY place a skill name becomes a path, so
- the safety check here is what guarantees that no name — freshly requested or read back from the
- (possibly corrupt) state file — can escape SKILLS_ROOT during delete/copy.
-
- Returns None for an unsafe name (empty, absolute, containing a path separator, "." / "..", or the
- reserved state-file name) so each caller can skip it per its own failure contract instead of
- touching an out-of-tree path. The state-file name is reserved because a skill whose name equals it
- would resolve to STATE_PATH and let a skill install clobber the reconciliation state.
- """
- if not name or name in (".", "..") or "/" in name or "\\" in name or name == STATE_FILE_NAME:
- return None
- return SKILLS_ROOT / name
-
-
-class GitSkill(BaseModel):
- """A git skill to materialize: `name`, `url`, `path`, and mount `ref` (branch, tag, or object id).
- This script resolves `ref` via `git ls-remote` in the sandbox, then sparse-fetches the resolved
- object id. The url and ref are format-validated here (defense in depth) so an unexpected value
- can't be interpreted as a git option or reach a non-public host."""
-
- model_config = ConfigDict(extra="ignore")
-
- name: str
- url: str
- path: str = ""
- # Branch, tag, or full object id from the agent_spec mount (resolved in-sandbox before fetch).
- ref: str
-
- @field_validator("url")
- @classmethod
- def _validate_url(cls, v: str) -> str:
- # Only the two public hosts the gateway supports. Anchoring on the "https://<host>/" prefix
- # blocks non-https schemes (file://, ext::, ssh) and any other host, so this URL can't be
- # turned into an SSRF/local-file/command vector once it reaches `git`.
- if not (v.startswith("https://github.com/") or v.startswith("https://gitlab.com/")):
- raise ValueError(f"git skill url must be a github.com/gitlab.com https URL: {v!r}")
- return v
-
- @field_validator("ref")
- @classmethod
- def _validate_ref(cls, v: str) -> str:
- if not v or ".." in v.split("/") or not GIT_REF_RE.match(v):
- raise ValueError(f"git skill ref is invalid: {v!r}")
- if v.replace("/", "") == "":
- raise ValueError(f"git skill ref must not consist only of slashes: {v!r}")
- return v
-
- @field_validator("path")
- @classmethod
- def _validate_path(cls, v: str) -> str:
- # Relative subpath within the repo. Reject absolute paths and ".." segments so it can't
- # escape the repo root. Leading/trailing slashes are stripped (repo root == ""). A leading
- # "-" is safe because subdirs are fed to `git sparse-checkout set` via stdin, never as args.
- if v.startswith("/"):
- raise ValueError(f"git skill path must be relative: {v!r}")
- stripped = v.strip("/")
- if stripped and ".." in stripped.split("/"):
- raise ValueError(f"git skill path must not contain '..': {v!r}")
- return stripped
-
-
-class DownloadedGitSkill(BaseModel):
- """A git skill recorded on disk: its dir name, the object id it was installed at, and the repo
- path it was installed from. The ref lets a later run skip re-downloading when the requested
- pin is unchanged; the path is part of the identity so that changing the path at the same
- pin still triggers a re-download instead of serving the previously-installed layout."""
-
- model_config = ConfigDict(extra="ignore")
-
- name: str
- # Empty when migrated from a pre-pin-tracking state file — treated as "unknown", forcing a
- # one-time re-download while still keeping the name tracked for pruning.
- ref: str = ""
- # Repo path the skill was installed from (repo root == ""). Empty on entries written before
- # path tracking; a mismatch against the requested path forces a one-time re-download.
- path: str = ""
-
-
-class SkillDownloaderState(BaseModel):
- """Persisted next to skills; tracks what was downloaded so undesired skills can be pruned."""
-
- model_config = ConfigDict(extra="ignore")
-
- downloaded_git_skills: list[DownloadedGitSkill] = Field(
- default_factory=list,
- description="Git skills successfully downloaded (name + installed object id).",
- )
-
-
-def _migrate_downloaded_git_skills(entries: list[object]) -> list[dict[str, object]]:
- """Normalize state entries: rename legacy `commit_sha` → `ref` and `subdir` → `path`."""
- migrated: list[dict[str, object]] = []
- for item in entries:
- if not isinstance(item, dict):
- continue
- entry = dict(item)
- if "ref" not in entry and isinstance(entry.get("commit_sha"), str):
- entry["ref"] = entry["commit_sha"]
- entry.pop("commit_sha", None)
- if "path" not in entry and isinstance(entry.get("subdir"), str):
- entry["path"] = entry["subdir"]
- entry.pop("subdir", None)
- migrated.append(entry)
- return migrated
-
-
-def load_state() -> SkillDownloaderState:
- if not STATE_PATH.is_file():
- return SkillDownloaderState()
- try:
- raw = json.loads(STATE_PATH.read_text(encoding="utf-8"))
- except (OSError, ValueError):
- return SkillDownloaderState()
- if not isinstance(raw, dict):
- return SkillDownloaderState()
- # Migrate a pre-pin-tracking state file: its `downloaded_git_names` (list of names, no pin)
- # becomes entries with an empty ref. That keeps the names tracked so undesired skills are
- # still pruned, and forces a one-time re-download (empty != any requested pin). Skipped when
- # the new field is already present so we never clobber real pins.
- if "downloaded_git_skills" not in raw and isinstance(raw.get("downloaded_git_names"), list):
- raw["downloaded_git_skills"] = [
- {"name": n, "ref": ""} for n in raw["downloaded_git_names"] if isinstance(n, str)
- ]
- if isinstance(raw.get("downloaded_git_skills"), list):
- raw["downloaded_git_skills"] = _migrate_downloaded_git_skills(raw["downloaded_git_skills"])
- try:
- return SkillDownloaderState.model_validate(raw)
- except ValidationError:
- return SkillDownloaderState()
-
-
-def save_state(state: SkillDownloaderState) -> None:
- STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
- tmp = STATE_PATH.with_name(STATE_PATH.name + ".tmp")
- tmp.write_text(state.model_dump_json(indent=2), encoding="utf-8")
- os.replace(tmp, STATE_PATH)
-
-
-def _rmtree(path: Path, ignore_errors: bool = True) -> None:
- try:
- shutil.rmtree(path, ignore_errors=ignore_errors)
- except OSError:
- pass
-
-
-def _delete_skill_dir_by_name(name: str, label: str) -> None:
- """Fail-closed prune of a no-longer-desired skill's directory: abort the whole run if removal
- fails so we never silently leave a stale skill on disk. skill_dir() returns None for a
- corrupt/tampered name, so rmtree never runs on an out-of-tree path."""
- dir_ = skill_dir(name)
- if dir_ is None or not dir_.is_dir():
- return
- try:
- shutil.rmtree(dir_)
- except OSError as e:
- sys.exit(f"Could not remove skill directory {dir_} ({label}): {e}")
-
-
-def git_skills_from_env(raw: str | None) -> list[GitSkill]:
- """Empty or missing env means "no desired git skills" (prune all git downloads)."""
- if raw is None or not str(raw).strip():
- return []
- try:
- data = json.loads(base64.b64decode(raw))
- except (ValueError, json.JSONDecodeError) as e:
- sys.exit(f"AGENT_GIT_SKILLS is not valid base64-encoded JSON: {e}")
- if not isinstance(data, list):
- sys.exit("AGENT_GIT_SKILLS must be a JSON list.")
- try:
- return [GitSkill.model_validate(item) for item in data]
- except ValidationError as e:
- sys.exit(f"Invalid AGENT_GIT_SKILLS entry: {e}")
-
-
-def reconcile_git_skills(git_skills: list[GitSkill], state: SkillDownloaderState) -> list[str]:
- """Delete dirs for git skills no longer desired; prune them from state. Returns removed names."""
- desired_names = {g.name for g in git_skills}
- removed: list[str] = []
- for entry in state.downloaded_git_skills:
- if entry.name in desired_names:
- continue
- # Fail-closed prune: a failed delete aborts the run rather than leaving a stale skill on disk.
- # The name is dropped from tracking either way.
- _delete_skill_dir_by_name(entry.name, entry.name)
- removed.append(entry.name)
- state.downloaded_git_skills = [
- e for e in state.downloaded_git_skills if e.name in desired_names
- ]
- return removed
-
-
-def _run_git(
- args: list[str], cwd: Path, input_text: str | None = None
-) -> subprocess.CompletedProcess[str]:
- """Run a git subcommand non-interactively and fail-closed on error.
-
- Args are passed as a list (never a shell string) and git runs with a scrubbed environment:
- ambient global/system config and credential helpers are ignored (so nothing can rewrite the URL
- via insteadOf or inject creds) and prompting is disabled (so a private/typo repo fails fast
- instead of hanging on a username prompt)."""
- env = {
- **os.environ,
- "GIT_TERMINAL_PROMPT": "0", # never prompt for credentials — fail fast on a non-public repo
- "GIT_CONFIG_NOSYSTEM": "1", # ignore /etc/gitconfig
- "GIT_CONFIG_GLOBAL": "/dev/null", # ignore ~/.gitconfig (insteadOf rewrites, cred helpers)
- "GCM_INTERACTIVE": "never",
- }
- try:
- proc = subprocess.run(
- ["git", *args],
- cwd=os.fspath(cwd),
- env=env,
- input=input_text,
- capture_output=True,
- text=True,
- timeout=GIT_CLONE_TIMEOUT_SECONDS,
- )
- except subprocess.TimeoutExpired:
- raise GitSkillError(f"git {args[0]} timed out after {GIT_CLONE_TIMEOUT_SECONDS}s")
- except OSError as e:
- raise GitSkillError(f"could not run git (is it installed in the sandbox?): {e}")
- if proc.returncode != 0:
- detail = (proc.stderr or proc.stdout or "").strip()[-500:]
- raise GitSkillError(f"git {args[0]} failed (exit {proc.returncode}): {detail}")
- return proc
-
-
-def _parse_ls_remote_object_id(stdout: str) -> str | None:
- """Prefer a peeled `ref^{}` line (annotated tags), else the first object id."""
- first: str | None = None
- for line in stdout.splitlines():
- trimmed = line.strip()
- if not trimmed:
- continue
- parts = trimmed.split(None, 1)
- sha = parts[0]
- name = parts[1] if len(parts) > 1 else ""
- if name.endswith("^{}"):
- return sha
- if first is None:
- first = sha
- return first
-
-
-def _resolve_object_id(url: str, ref: str, cwd: Path) -> str:
- """Resolve a mount ref (branch/tag/SHA) to an object id inside the sandbox.
-
- Full object ids are used as-is (`git ls-remote` does not advertise bare SHAs on
- GitHub/GitLab). Branches and tags are resolved via `git ls-remote` so a branch tip
- advances on later sandbox inits.
- """
- if OBJECT_ID_RE.match(ref):
- return ref
- proc = _run_git(["ls-remote", url, ref], cwd=cwd)
- oid = _parse_ls_remote_object_id(proc.stdout or "")
- if oid is None:
- raise GitSkillError(f"git ls-remote returned no commit for {url} ref {ref}")
- return oid
-
-
-def _git_fetch_repo(
- url: str, object_id: str, paths: list[str], parent_dir: Path
-) -> Path:
- """Object-id-pinned, shallow, blob-filtered sparse checkout of `paths` into a fresh repo under
- `parent_dir`, returning the repo working directory.
-
- When every requested path is the repo root ("") we can't sparse-restrict, so we do a full
- shallow checkout (all blobs). Otherwise we cone-sparse to just the requested paths and use
- --filter=blob:none so only those paths' blobs are fetched — the whole point of the approach.
- Paths are fed to `git sparse-checkout set` via stdin so a path value can never be read as an
- option/argument. Fetching by object id relies on the server allowing it, which github.com and
- gitlab.com both do."""
- repo = parent_dir / "repo"
- _run_git(["init", "-q", os.fspath(repo)], cwd=parent_dir)
- _run_git(["remote", "add", "origin", url], cwd=repo)
-
- cone_dirs = [s for s in dict.fromkeys(paths) if s] # de-duped, root ("") dropped
- want_full_tree = any(s == "" for s in paths)
- if not want_full_tree and cone_dirs:
- _run_git(["sparse-checkout", "init", "--cone"], cwd=repo)
- _run_git(
- ["sparse-checkout", "set", "--stdin"],
- cwd=repo,
- input_text="".join(f"{d}\n" for d in cone_dirs),
- )
-
- fetch_args = ["-c", "protocol.file.allow=never", "fetch", "-q", "--depth", "1"]
- if not want_full_tree:
- fetch_args.append("--filter=blob:none")
- fetch_args += ["origin", object_id]
- _run_git(fetch_args, cwd=repo)
-
- _run_git(
- ["-c", "advice.detachedHead=false", "checkout", "-q", "--detach", "FETCH_HEAD"], cwd=repo
- )
- return repo
-
-
-def _installed_size_bytes(path: Path) -> int:
- """Total size of regular files under `path`, excluding the repo's .git dir and without following
- symlinks (they're copied as links, not their targets), used to enforce GIT_SKILL_MAX_BYTES."""
- total = 0
- for root, dirs, files in os.walk(path, followlinks=False):
- if ".git" in dirs:
- dirs.remove(".git")
- for name in files:
- try:
- total += (Path(root) / name).lstat().st_size
- except OSError:
- continue
- return total
-
-
-def _install_git_skill(repo_root: Path, skill: GitSkill) -> None:
- dest = skill_dir(skill.name)
- if dest is None:
- raise GitSkillError(f"Git skill: unsafe directory name {skill.name!r}")
- repo_root_resolved = repo_root.resolve()
- src = (repo_root / skill.path).resolve() if skill.path else repo_root_resolved
- # Defense in depth: even though path is validated, ensure the resolved source stays within the
- # cloned repo (guards against a symlinked directory component pointing outside the checkout).
- if src != repo_root_resolved and not src.is_relative_to(repo_root_resolved):
- raise GitSkillError(
- f"Git skill {skill.name}: path '{skill.path}' escapes the repository"
- )
- if not src.is_dir():
- raise GitSkillError(
- f"Git skill {skill.name}: path '{skill.path}' not found in repository"
- )
- if _installed_size_bytes(src) > GIT_SKILL_MAX_BYTES:
- raise GitSkillError(
- f"Git skill {skill.name}: content exceeds {GIT_SKILL_MAX_BYTES} bytes"
- )
- # Stage the fresh copy into a temp sibling dir, then swap it in with an atomic rename. `dest` is
- # only touched once the copy has fully succeeded, so a failed/partial copytree can never leave
- # the skill missing (which would contradict the path the gateway advertises in the prompt) — the
- # previously installed copy stays intact on failure. `.git` is excluded (only present for a repo-
- # root skill) and symlinks are copied as links, never followed, so a link inside the repo can't
- # pull external file contents into the skill dir at copy time.
- staging = dest.with_name(f".{dest.name}.new-{os.getpid()}")
- _rmtree(staging) # clear any leftover staging dir from an earlier aborted run
- try:
- shutil.copytree(src, staging, symlinks=True, ignore=shutil.ignore_patterns(".git"))
- except (OSError, shutil.Error) as e:
- # shutil.Error (a multi-file copy error, NOT an OSError subclass) must be caught here too so a
- # bad copy is reported as a per-skill GitSkillError (WARNING + counted failure) instead of an
- # uncaught traceback that aborts the whole run and skips save_state for the group.
- _rmtree(staging)
- raise GitSkillError(f"Failed to stage git skill {skill.name}: {e}")
- try:
- if dest.exists():
- shutil.rmtree(dest)
- # Same filesystem (both under SKILLS_ROOT), so this is an atomic rename; the window where the
- # skill dir is absent shrinks from a full copy to a single near-instant rename.
- os.replace(staging, dest)
- except OSError as e:
- _rmtree(staging)
- raise GitSkillError(f"Failed to install git skill {skill.name}: {e}")
-
-
-def _mark_git_downloaded(
- state: SkillDownloaderState, skill: GitSkill, object_id: str
-) -> None:
- """Upsert a git skill's installed object id + path so the next run can skip when both match."""
- for entry in state.downloaded_git_skills:
- if entry.name == skill.name:
- entry.ref = object_id
- entry.path = skill.path
- return
- state.downloaded_git_skills.append(
- DownloadedGitSkill(name=skill.name, ref=object_id, path=skill.path)
- )
-
-
-def _git_skill_already_installed(
- state: SkillDownloaderState, skill: GitSkill, object_id: str
-) -> bool:
- """True only when the skill is recorded at the resolved object id AND path AND its directory is
- still present. Re-download otherwise (branch tip moved, path changed, first install, or the
- dir was removed out-of-band) so the prompt never points at a missing/stale skill dir."""
- dir_ = skill_dir(skill.name)
- if dir_ is None or not dir_.is_dir():
- return False
- return any(
- e.name == skill.name and e.ref == object_id and e.path == skill.path
- for e in state.downloaded_git_skills
- )
-
-
-def download_git_skills(
- git_skills: list[GitSkill], state: SkillDownloaderState
-) -> int:
- """Ensure desired git skills are installed at their resolved object id, skipping any already
- present at that pin so an unchanged tip no longer re-clones every run.
-
- Ref resolution (`git ls-remote`) runs here in the sandbox — the host never spawns git.
- Failures are isolated per repo group and per skill so one bad repo/subdir doesn't prevent the
- rest from being attempted; each failure is logged to stderr. Returns the number of skills that are
- satisfied this run — freshly installed OR already present at the resolved pin (never raises).
- The caller (run_git_download) compares this to the number requested and exits non-zero if any are
- missing, so the overall behaviour is fail-closed."""
- SKILLS_ROOT.mkdir(parents=True, exist_ok=True)
-
- # Resolve mount refs → object ids, then skip skills already installed at that pin.
- pending: list[tuple[GitSkill, str]] = []
- satisfied = 0
- for skill in git_skills:
- try:
- object_id = _resolve_object_id(skill.url, skill.ref, SKILLS_ROOT)
- except GitSkillError as e:
- print(f"WARNING: {e} (skill: {skill.name})", file=sys.stderr)
- continue
- if _git_skill_already_installed(state, skill, object_id):
- satisfied += 1
- else:
- pending.append((skill, object_id))
-
- # Group pending by (repo, object id) so a repo shared by several skills is cloned once and its
- # needed paths fetched together in a single sparse checkout.
- skills_by_repo: dict[tuple[str, str], list[tuple[GitSkill, str]]] = {}
- for skill, object_id in pending:
- skills_by_repo.setdefault((skill.url, object_id), []).append((skill, object_id))
-
- for (url, object_id), group in skills_by_repo.items():
- try:
- with tempfile.TemporaryDirectory(
- prefix=".git-skill-dl-",
- dir=os.fspath(SKILLS_ROOT),
- ignore_cleanup_errors=True,
- ) as tmp_str:
- repo_root = _git_fetch_repo(
- url,
- object_id,
- [s.path for s, _ in group],
- Path(tmp_str),
- )
- # Install each skill independently so a bad path only skips that one skill.
- for skill, oid in group:
- try:
- _install_git_skill(repo_root, skill)
- _mark_git_downloaded(state, skill, oid)
- satisfied += 1
- except GitSkillError as e:
- print(f"WARNING: {e}", file=sys.stderr)
- except GitSkillError as e:
- # Collect (don't abort-on-first) so EVERY failing skill is reported before the run exits;
- # run_git_download turns a non-zero failed count into a non-zero exit (fail-closed), which
- # fails the whole agent request. We still don't purge a previously installed copy here, so
- # a reused sandbox keeps the old files on disk even though this run is about to fail.
- names = ", ".join(s.name for s, _ in group)
- print(f"WARNING: {e} (skills: {names})", file=sys.stderr)
- save_state(state)
- return satisfied
-
-
-def run_git_download() -> None:
- git_skills = git_skills_from_env(os.environ.get("AGENT_GIT_SKILLS"))
- state = load_state()
- removed = reconcile_git_skills(git_skills, state)
- save_state(state) # persist pruning even when there is nothing to download
-
- if not git_skills:
- if removed:
- print(f"Removed {len(removed)} git skill(s) from disk. (AGENT_GIT_SKILLS empty.)")
- return
-
- satisfied = download_git_skills(git_skills, state)
- failed = len(git_skills) - satisfied
- suffix = f" Removed {len(removed)} stale git skill(s)." if removed else ""
- if failed:
- # Fail-closed: a requested skill that couldn't be installed exits non-zero so the gateway
- # (ensureExecSuccess) fails the whole agent request, matching the fail-closed SKILL.md fetch.
- # Per-skill WARNINGs above list exactly what failed.
- sys.exit(
- f"Failed to install {failed}/{len(git_skills)} git skill(s); see warnings above.{suffix}"
- )
- print(f"Ensured {satisfied} git skill(s) (downloaded or already up to date).{suffix}")
-
-
-def main() -> None:
- run_git_download()
-
-
-if __name__ == "__main__":
- main()
diff --git a/packages/trueforge-core/src/core/sandbox/scripts/skill_downloader.py b/packages/trueforge-core/src/core/sandbox/scripts/skill_downloader.py
new file mode 100644
index 000000000..925653fb4
--- /dev/null
+++ b/packages/trueforge-core/src/core/sandbox/scripts/skill_downloader.py
@@ -0,0 +1,730 @@
+#!/usr/bin/env python3
+
+# /// script
+# dependencies = ["pydantic==2.12.5"]
+# ///
+
+"""Materialize skill directories into TFY_SKILLS_DIR (default /opt/tfy/skills).
+
+Reads `.tfy-requested-skills.json` (uploaded by the host), installs each entry under
+TFY_SKILLS_DIR, then deletes the requested file. Empty `"skills": []` clears installs.
+On-disk state skips unchanged skills and removes ones no longer requested.
+
+Git: each skill is a blob-filtered sparse `git` clone of only the requested subdir at
+a resolved object id (keyed by name). Mount `ref` (branch, tag, or full object id) is
+resolved here via `git ls-remote` — the host never spawns git. Sparse clone is used
+instead of a full-repo tarball so cost tracks subdir size, not monorepo size.
+
+Registry: download a presigned tar of the skill artifact and extract it under the
+skill name. State records installed FQNs for skip/remove.
+
+Requested file example:
+ {
+ "skills": [
+ {
+ "type": "git",
+ "name": "echo",
+ "url": "https://github.com/acme/skills.git",
+ "path": "echo",
+ "ref": "main"
+ },
+ {
+ "type": "registry",
+ "name": "echo",
+ "fqn": "agent-skill:acme/team-a/echo:3",
+ "presigned_url": "https://..."
+ }
+ ]
+ }
+
+State file example (git `ref` is the resolved object id used for skip/reinstall, not the
+requested pin like `"main"`):
+ {
+ "downloaded_git_skills": [
+ {
+ "name": "echo",
+ "ref": "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15",
+ "path": "echo"
+ }
+ ],
+ "downloaded_registry_skills": [
+ {
+ "fqn": "agent-skill:acme/team-a/echo:3",
+ "name": "echo"
+ }
+ ]
+ }
+
+Git skill fields (requested file):
+ - name: install directory name under TFY_SKILLS_DIR
+ - url: https://github.com/... or https://gitlab.com/...
+ - path: subdir in the repo (empty = repo root)
+ - ref: branch, tag, or full object id (resolved to an oid before fetch; state stores that oid)
+
+Registry skill fields (requested file):
+ - name: install directory name under TFY_SKILLS_DIR
+ - fqn: version-pinned registry FQN
+ - presigned_url: temporary download URL for the skill tar
+
+Env:
+ - TFY_SKILLS_DIR: skills directory (default /opt/tfy/skills).
+"""
+
+from __future__ import annotations
+
+import errno
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Annotated, Any, ClassVar, Literal
+from abc import ABC, abstractmethod
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
+
+DEFAULT_SKILLS_DIR = "/opt/tfy/skills"
+STATE_FILE_NAME = ".tfy-skill-downloader-state.json"
+REQUESTED_FILE_NAME = ".tfy-requested-skills.json"
+# Per-git-invocation wall-clock cap so a hung/slow fetch can't stall sandbox init indefinitely.
+GIT_CLONE_TIMEOUT_SECONDS = 120
+# Per-registry-download wall-clock cap so a hung/slow download can't stall sandbox init indefinitely.
+REGISTRY_DOWNLOAD_TIMEOUT_SECONDS = 120
+# Cap registry download bytes and post-extract installed size (same bound as git installs).
+REGISTRY_SKILL_MAX_BYTES = 100 * 1024 * 1024 # 100MB
+# Cap the installed skill size so a huge repo-root skill can't fill the persistent skills dir. The
+# sparse clone already bounds a subdir skill to its subdir; this also guards the whole-repo (root
+# subdir) case where checkout hydrates every file.
+GIT_SKILL_MAX_BYTES = 100 * 1024 * 1024 # 100MB installed
+# Resolved object id used for fetch + skip state (sha1 / sha256).
+OBJECT_ID_RE = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$")
+# Mount ref (branch/tag/SHA) charset — must match host AgentSpec SkillMount.ref validation.
+# Validated before the value is passed to `git ls-remote` / used as a fetch arg fallback.
+GIT_REF_RE = re.compile(r"^[A-Za-z0-9._\-/]+$")
+SKILLS_ROOT = Path(os.environ.get("TFY_SKILLS_DIR", DEFAULT_SKILLS_DIR))
+STATE_PATH = SKILLS_ROOT / STATE_FILE_NAME
+
+
+class GitSkillError(Exception):
+ """Recoverable per-repo/per-skill git failure. Raised (not sys.exit) so a single bad repo or
+ subdirectory doesn't abort the remaining repository groups (best-effort failure isolation)."""
+
+
+def skill_dir(name: str) -> Path | None:
+ """Map skill name → path under SKILLS_ROOT, or None if unsafe (incl. state/requested filenames)."""
+ if not name or name in (".", "..") or "/" in name or "\\" in name or name in (
+ STATE_FILE_NAME,
+ REQUESTED_FILE_NAME,
+ ):
+ return None
+ return SKILLS_ROOT / name
+
+
+class GitSkill(BaseModel):
+ """A git skill to materialize: `name`, `url`, `path`, and mount `ref` (branch, tag, or object id).
+ This script resolves `ref` via `git ls-remote` in the sandbox, then sparse-fetches the resolved
+ object id. The url and ref are format-validated here (defense in depth) so an unexpected value
+ can't be interpreted as a git option or reach a non-public host."""
+
+ model_config = ConfigDict(extra="ignore")
+
+ name: str
+ url: str
+ path: str = ""
+ # Branch, tag, or full object id from the agent_spec mount (resolved in-sandbox before fetch).
+ ref: str
+
+ @field_validator("url")
+ @classmethod
+ def _validate_url(cls, v: str) -> str:
+ # Only the two public hosts the gateway supports. Anchoring on the "https://<host>/" prefix
+ # blocks non-https schemes (file://, ext::, ssh) and any other host, so this URL can't be
+ # turned into an SSRF/local-file/command vector once it reaches `git`.
+ if not (v.startswith("https://github.com/") or v.startswith("https://gitlab.com/")):
+ raise ValueError(f"git skill url must be a github.com/gitlab.com https URL: {v!r}")
+ return v
+
+ @field_validator("ref")
+ @classmethod
+ def _validate_ref(cls, v: str) -> str:
+ if not v or ".." in v.split("/") or not GIT_REF_RE.match(v):
+ raise ValueError(f"git skill ref is invalid: {v!r}")
+ if v.replace("/", "") == "":
+ raise ValueError(f"git skill ref must not consist only of slashes: {v!r}")
+ return v
+
+ @field_validator("path")
+ @classmethod
+ def _validate_path(cls, v: str) -> str:
+ # Relative subpath within the repo. Reject absolute paths and ".." segments so it can't
+ # escape the repo root. Leading/trailing slashes are stripped (repo root == ""). A leading
+ # "-" is safe because subdirs are fed to `git sparse-checkout set` via stdin, never as args.
+ if v.startswith("/"):
+ raise ValueError(f"git skill path must be relative: {v!r}")
+ stripped = v.strip("/")
+ if stripped and ".." in stripped.split("/"):
+ raise ValueError(f"git skill path must not contain '..': {v!r}")
+ return stripped
+
+
+class DownloadedGitSkill(BaseModel):
+ """A git skill recorded on disk: its dir name, the object id it was installed at, and the repo
+ path it was installed from. The ref lets a later run skip re-downloading when the requested
+ pin is unchanged; the path is part of the identity so that changing the path at the same
+ pin still triggers a re-download instead of serving the previously-installed layout."""
+
+ model_config = ConfigDict(extra="ignore")
+
+ name: str
+ # Installed object id; empty = unknown → reinstall. Name still tracked for cleanup.
+ ref: str = ""
+ # Installed repo path ("" = root); mismatch vs requested forces reinstall.
+ path: str = ""
+
+
+class DownloadedRegistrySkill(BaseModel):
+ """A registry skill recorded on disk: its FQN and name."""
+
+ model_config = ConfigDict(extra="ignore")
+
+ fqn: str
+ name: str
+
+
+class SkillDownloaderState(BaseModel):
+ model_config = ConfigDict(extra="ignore")
+
+ # Keep git and registry installs in separate lists (do not merge into one discriminated
+ # array). Each SkillSource already reads/writes only its own rows, and the persisted
+ # identities differ: git skip/reinstall keys on name + resolved object id + path;
+ # registry keys on fqn + name. A shared list would add type filters with no load-path win.
+ downloaded_git_skills: list[DownloadedGitSkill] = Field(default_factory=list)
+ downloaded_registry_skills: list[DownloadedRegistrySkill] = Field(default_factory=list)
+
+
+class RequestedGitSkill(BaseModel):
+ model_config = ConfigDict(extra="ignore")
+
+ type: Literal["git"] = "git"
+ name: str
+ url: str
+ path: str = ""
+ ref: str
+
+ def resolve(self) -> GitSkill:
+ try:
+ return GitSkill(name=self.name, url=self.url, path=self.path, ref=self.ref)
+ except ValidationError as e:
+ sys.exit(f"Invalid git entry in {REQUESTED_FILE_NAME}: {e}")
+
+
+class RequestedRegistrySkill(BaseModel):
+ model_config = ConfigDict(extra="ignore")
+
+ type: Literal["registry"] = "registry"
+ name: str
+ fqn: str = Field(min_length=1)
+ presigned_url: str = Field(min_length=1)
+
+ def resolve(self) -> RequestedRegistrySkill:
+ if skill_dir(self.name) is None:
+ sys.exit(f"Invalid registry skill name: {self.name!r}")
+ return self
+
+
+RequestedSkill = RequestedGitSkill | RequestedRegistrySkill
+
+
+class RequestedSkillsFileContent(BaseModel):
+ model_config = ConfigDict(extra="ignore")
+
+ skills: list[Annotated[RequestedSkill, Field(discriminator="type")]] = Field(
+ default_factory=list
+ )
+
+
+def load_state() -> SkillDownloaderState:
+ if not STATE_PATH.is_file():
+ return SkillDownloaderState()
+ try:
+ return SkillDownloaderState.model_validate_json(STATE_PATH.read_text(encoding="utf-8"))
+ except (OSError, ValueError, ValidationError):
+ # ValueError covers UnicodeDecodeError from corrupt/non-UTF-8 state in agent-writable skills dir.
+ return SkillDownloaderState()
+
+
+def save_state(state: SkillDownloaderState) -> None:
+ STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
+ tmp = STATE_PATH.with_name(STATE_PATH.name + ".tmp")
+ tmp.write_text(state.model_dump_json(indent=2), encoding="utf-8")
+ os.replace(tmp, STATE_PATH)
+
+
+def _rmtree(path: Path, ignore_errors: bool = True) -> None:
+ try:
+ shutil.rmtree(path, ignore_errors=ignore_errors)
+ except OSError:
+ pass
+
+
+def _format_os_error(e: BaseException) -> str:
+ """Prefer a clear sandbox-disk-full message when errno is ENOSPC."""
+ if isinstance(e, OSError) and e.errno == errno.ENOSPC:
+ return f"sandbox disk full: {e}"
+ return str(e)
+
+
+def _delete_skill_dir_by_name(name: str, label: str) -> None:
+ """Remove a no-longer-requested skill directory; abort the whole run if removal
+ fails so we never silently leave a stale skill on disk. skill_dir() returns None for a
+ corrupt/tampered name, so rmtree never runs on an out-of-tree path."""
+ dir_ = skill_dir(name)
+ if dir_ is None or not dir_.is_dir():
+ return
+ try:
+ shutil.rmtree(dir_)
+ except OSError as e:
+ sys.exit(f"Could not remove skill directory {dir_} ({label}): {e}")
+
+
+
+def _installed_size_bytes(path: Path) -> int:
+ """Total size of regular files under `path`, excluding `.git`, without following symlinks."""
+ total = 0
+ for root, dirs, files in os.walk(path, followlinks=False):
+ if ".git" in dirs:
+ dirs.remove(".git")
+ for name in files:
+ try:
+ total += (Path(root) / name).lstat().st_size
+ except OSError:
+ continue
+ return total
+
+
+def _delete_requested_file() -> None:
+ path = SKILLS_ROOT / REQUESTED_FILE_NAME
+ try:
+ if path.is_file():
+ path.unlink()
+ except OSError as e:
+ print(f"WARNING: could not delete {REQUESTED_FILE_NAME}: {e}", file=sys.stderr)
+
+
+
+class SkillSource(ABC):
+ """One install source (git clone, registry tar, …). Helpers live on subclasses."""
+
+ type: ClassVar[str]
+
+ @abstractmethod
+ def reconcile(self, skills: list[Any], state: SkillDownloaderState) -> list[str]:
+ """Remove installs no longer requested. Returns removed ids/names."""
+
+ @abstractmethod
+ def download(self, skills: list[Any], state: SkillDownloaderState) -> tuple[int, int]:
+ """Install/skip requested skills. If already present, skip. Returns (installed, skipped)."""
+
+
+class RegistrySkillSource(SkillSource):
+ type = "registry"
+
+ def reconcile(self, skills: list[Any], state: SkillDownloaderState) -> list[str]:
+ requested_fqns = {s.fqn for s in skills}
+ removed: list[str] = []
+ for entry in list(state.downloaded_registry_skills):
+ if entry.fqn in requested_fqns:
+ continue
+ _delete_skill_dir_by_name(entry.name, label=f"stale registry fqn {entry.fqn}")
+ removed.append(entry.fqn)
+ state.downloaded_registry_skills = [e for e in state.downloaded_registry_skills if e.fqn in requested_fqns]
+ return removed
+
+ def download(self, skills: list[Any], state: SkillDownloaderState) -> tuple[int, int]:
+ installed = 0
+ skipped = 0
+ for skill in skills:
+ if self._already_installed(state, skill):
+ skipped += 1
+ continue
+ try:
+ self._install_tar(skill)
+ self._mark_downloaded(state, skill)
+ installed += 1
+ except RuntimeError as e:
+ print(f"WARNING: {e}", file=sys.stderr)
+ return installed, skipped
+
+ def _already_installed(self, state: SkillDownloaderState, skill: RequestedRegistrySkill) -> bool:
+ dir_ = skill_dir(skill.name)
+ if dir_ is None or not dir_.is_dir():
+ return False
+ return any(e.fqn == skill.fqn and e.name == skill.name for e in state.downloaded_registry_skills)
+
+ def _mark_downloaded(self, state: SkillDownloaderState, skill: RequestedRegistrySkill) -> None:
+ for entry in state.downloaded_registry_skills:
+ if entry.fqn == skill.fqn:
+ if entry.name != skill.name:
+ _delete_skill_dir_by_name(
+ entry.name, label=f"renamed registry fqn {entry.fqn}"
+ )
+ entry.name = skill.name
+ return
+ state.downloaded_registry_skills.append(DownloadedRegistrySkill(fqn=skill.fqn, name=skill.name))
+
+ def _install_tar(self, skill: RequestedRegistrySkill) -> None:
+ """Stage extract under SKILLS_ROOT, then swap over the prior dir."""
+ dir_ = skill_dir(skill.name)
+ if dir_ is None:
+ raise RuntimeError(f"invalid skill name {skill.name!r}")
+ try:
+ with urllib.request.urlopen(
+ skill.presigned_url, timeout=REGISTRY_DOWNLOAD_TIMEOUT_SECONDS
+ ) as resp:
+ data = resp.read(REGISTRY_SKILL_MAX_BYTES + 1)
+ except (urllib.error.URLError, TimeoutError, OSError) as e:
+ raise RuntimeError(f"download failed for {skill.fqn}: {_format_os_error(e)}") from e
+ if len(data) > REGISTRY_SKILL_MAX_BYTES:
+ raise RuntimeError(f"tar for {skill.fqn} exceeds {REGISTRY_SKILL_MAX_BYTES} bytes")
+
+ staging = dir_.with_name(f".{dir_.name}.new-{os.getpid()}")
+ _rmtree(staging)
+ try:
+ staging.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(
+ prefix=".registry-skill-tar-",
+ dir=os.fspath(SKILLS_ROOT),
+ ignore_cleanup_errors=True,
+ ) as tmp:
+ tar_path = Path(tmp) / "skill.tar"
+ tar_path.write_bytes(data)
+ with tarfile.open(tar_path, mode="r:*") as tar:
+ # The download cap only limits compressed bytes. Member `.size` is the
+ # uncompressed file length from the tar headers — sum it and refuse before
+ # extractall so a small archive cannot expand past REGISTRY_SKILL_MAX_BYTES on disk.
+ claimed = sum(m.size for m in tar.getmembers() if m.isreg())
+ if claimed > REGISTRY_SKILL_MAX_BYTES:
+ raise RuntimeError(
+ f"extracted skill {skill.fqn} exceeds {REGISTRY_SKILL_MAX_BYTES} bytes"
+ )
+ # Registry skill tars are system-generated (not user-uploaded), so extractall is fine.
+ # filter= is 3.12+; local TFY provider often runs host 3.10/3.11.
+ if sys.version_info >= (3, 12):
+ tar.extractall(path=staging, filter="data")
+ else:
+ tar.extractall(path=staging)
+ if dir_.exists():
+ shutil.rmtree(dir_)
+ os.replace(staging, dir_)
+ except RuntimeError:
+ _rmtree(staging)
+ raise
+ except (tarfile.TarError, OSError) as e:
+ _rmtree(staging)
+ raise RuntimeError(f"extract failed for {skill.fqn}: {_format_os_error(e)}") from e
+
+
+class GitSkillSource(SkillSource):
+ type = "git"
+
+ def reconcile(self, skills: list[Any], state: SkillDownloaderState) -> list[str]:
+ requested_names = {g.name for g in skills}
+ removed: list[str] = []
+ for entry in state.downloaded_git_skills:
+ if entry.name in requested_names:
+ continue
+ # Fail closed: a failed delete aborts the run rather than leaving a stale skill on disk.
+ _delete_skill_dir_by_name(entry.name, entry.name)
+ removed.append(entry.name)
+ state.downloaded_git_skills = [
+ e for e in state.downloaded_git_skills if e.name in requested_names
+ ]
+ return removed
+
+ def download(self, skills: list[Any], state: SkillDownloaderState) -> tuple[int, int]:
+ """Install requested git skills at resolved object ids; skip unchanged pins.
+
+ Failures are isolated per repo/skill (stderr warnings). Returns (installed, skipped);
+ caller exits non-zero if any requested skill is missing.
+ """
+ SKILLS_ROOT.mkdir(parents=True, exist_ok=True)
+
+ pending: list[tuple[GitSkill, str]] = []
+ installed = 0
+ skipped = 0
+ for skill in skills:
+ try:
+ object_id = self._resolve_object_id(skill.url, skill.ref, SKILLS_ROOT)
+ except GitSkillError as e:
+ print(f"WARNING: {e} (skill: {skill.name})", file=sys.stderr)
+ continue
+ if self._already_installed(state, skill, object_id):
+ skipped += 1
+ else:
+ pending.append((skill, object_id))
+
+ skills_by_repo: dict[tuple[str, str], list[tuple[GitSkill, str]]] = {}
+ for skill, object_id in pending:
+ skills_by_repo.setdefault((skill.url, object_id), []).append((skill, object_id))
+
+ for (url, object_id), group in skills_by_repo.items():
+ try:
+ with tempfile.TemporaryDirectory(
+ prefix=".git-skill-dl-",
+ dir=os.fspath(SKILLS_ROOT),
+ ignore_cleanup_errors=True,
+ ) as tmp_str:
+ repo_root = self._fetch_repo(
+ url,
+ object_id,
+ [s.path for s, _ in group],
+ Path(tmp_str),
+ )
+ for skill, oid in group:
+ try:
+ self._install_skill(repo_root, skill)
+ self._mark_downloaded(state, skill, oid)
+ installed += 1
+ except GitSkillError as e:
+ print(f"WARNING: {e}", file=sys.stderr)
+ except GitSkillError as e:
+ # Don't abort-on-first: report every failing group; keep prior installs on disk.
+ names = ", ".join(s.name for s, _ in group)
+ print(f"WARNING: {e} (skills: {names})", file=sys.stderr)
+ return installed, skipped
+
+ def _run_git(
+ self, args: list[str], cwd: Path, input_text: str | None = None
+ ) -> subprocess.CompletedProcess[str]:
+ """Run a git subcommand non-interactively and fail-closed on error."""
+ env = {
+ **os.environ,
+ "GIT_TERMINAL_PROMPT": "0",
+ "GIT_CONFIG_NOSYSTEM": "1",
+ "GIT_CONFIG_GLOBAL": "/dev/null",
+ "GCM_INTERACTIVE": "never",
+ }
+ try:
+ proc = subprocess.run(
+ ["git", *args],
+ cwd=os.fspath(cwd),
+ env=env,
+ input=input_text,
+ capture_output=True,
+ text=True,
+ timeout=GIT_CLONE_TIMEOUT_SECONDS,
+ )
+ except subprocess.TimeoutExpired:
+ raise GitSkillError(f"git {args[0]} timed out after {GIT_CLONE_TIMEOUT_SECONDS}s")
+ except OSError as e:
+ raise GitSkillError(f"could not run git (is it installed in the sandbox?): {e}")
+ if proc.returncode != 0:
+ detail = (proc.stderr or proc.stdout or "").strip()[-500:]
+ raise GitSkillError(f"git {args[0]} failed (exit {proc.returncode}): {detail}")
+ return proc
+
+ def _parse_ls_remote_object_id(self, stdout: str) -> str | None:
+ """Prefer a peeled `ref^{}` line (annotated tags), else the first object id."""
+ first: str | None = None
+ for line in stdout.splitlines():
+ trimmed = line.strip()
+ if not trimmed:
+ continue
+ parts = trimmed.split(None, 1)
+ sha = parts[0]
+ name = parts[1] if len(parts) > 1 else ""
+ if name.endswith("^{}"):
+ return sha
+ if first is None:
+ first = sha
+ return first
+
+ def _resolve_object_id(self, url: str, ref: str, cwd: Path) -> str:
+ """Resolve a mount ref (branch/tag/SHA) to an object id inside the sandbox."""
+ if OBJECT_ID_RE.match(ref):
+ return ref
+ proc = self._run_git(["ls-remote", url, ref], cwd=cwd)
+ oid = self._parse_ls_remote_object_id(proc.stdout or "")
+ if oid is None:
+ raise GitSkillError(f"git ls-remote returned no commit for {url} ref {ref}")
+ return oid
+
+ def _fetch_repo(
+ self, url: str, object_id: str, paths: list[str], parent_dir: Path
+ ) -> Path:
+ """Object-id-pinned, shallow, blob-filtered sparse checkout of `paths`."""
+ repo = parent_dir / "repo"
+ self._run_git(["init", "-q", os.fspath(repo)], cwd=parent_dir)
+ self._run_git(["remote", "add", "origin", url], cwd=repo)
+
+ cone_dirs = [s for s in dict.fromkeys(paths) if s]
+ want_full_tree = any(s == "" for s in paths)
+ if not want_full_tree and cone_dirs:
+ self._run_git(["sparse-checkout", "init", "--cone"], cwd=repo)
+ self._run_git(
+ ["sparse-checkout", "set", "--stdin"],
+ cwd=repo,
+ input_text="".join(f"{d}\n" for d in cone_dirs),
+ )
+
+ fetch_args = ["-c", "protocol.file.allow=never", "fetch", "-q", "--depth", "1"]
+ if not want_full_tree:
+ fetch_args.append("--filter=blob:none")
+ fetch_args += ["origin", object_id]
+ self._run_git(fetch_args, cwd=repo)
+
+ self._run_git(
+ ["-c", "advice.detachedHead=false", "checkout", "-q", "--detach", "FETCH_HEAD"],
+ cwd=repo,
+ )
+ return repo
+
+ def _install_skill(self, repo_root: Path, skill: GitSkill) -> None:
+ dest = skill_dir(skill.name)
+ if dest is None:
+ raise GitSkillError(f"Git skill: unsafe directory name {skill.name!r}")
+ repo_root_resolved = repo_root.resolve()
+ src = (repo_root / skill.path).resolve() if skill.path else repo_root_resolved
+ if src != repo_root_resolved and not src.is_relative_to(repo_root_resolved):
+ raise GitSkillError(
+ f"Git skill {skill.name}: path '{skill.path}' escapes the repository"
+ )
+ if not src.is_dir():
+ raise GitSkillError(
+ f"Git skill {skill.name}: path '{skill.path}' not found in repository"
+ )
+ if _installed_size_bytes(src) > GIT_SKILL_MAX_BYTES:
+ raise GitSkillError(
+ f"Git skill {skill.name}: content exceeds {GIT_SKILL_MAX_BYTES} bytes"
+ )
+ staging = dest.with_name(f".{dest.name}.new-{os.getpid()}")
+ _rmtree(staging)
+ try:
+ shutil.copytree(src, staging, symlinks=True, ignore=shutil.ignore_patterns(".git"))
+ except (OSError, shutil.Error) as e:
+ _rmtree(staging)
+ raise GitSkillError(f"Failed to stage git skill {skill.name}: {_format_os_error(e)}")
+ try:
+ if dest.exists():
+ shutil.rmtree(dest)
+ os.replace(staging, dest)
+ except OSError as e:
+ _rmtree(staging)
+ raise GitSkillError(f"Failed to install git skill {skill.name}: {_format_os_error(e)}")
+
+ def _mark_downloaded(
+ self, state: SkillDownloaderState, skill: GitSkill, object_id: str
+ ) -> None:
+ for entry in state.downloaded_git_skills:
+ if entry.name == skill.name:
+ entry.ref = object_id
+ entry.path = skill.path
+ return
+ state.downloaded_git_skills.append(
+ DownloadedGitSkill(name=skill.name, ref=object_id, path=skill.path)
+ )
+
+ def _already_installed(
+ self, state: SkillDownloaderState, skill: GitSkill, object_id: str
+ ) -> bool:
+ dir_ = skill_dir(skill.name)
+ if dir_ is None or not dir_.is_dir():
+ return False
+ return any(
+ e.name == skill.name and e.ref == object_id and e.path == skill.path
+ for e in state.downloaded_git_skills
+ )
+
+
+
+# Add a third skill type:
+# 1. RequestedFooSkill + resolve() + union member (RequestedSkill)
+# 2. FooSkillSource with reconcile / download
+# 3. Append FooSkillSource() to SKILL_SOURCES (order = install order)
+# 4. Wire type: 'foo' on the host (SkillMounter)
+# Registry first: presigned URLs expire; git sparse clones can take much longer.
+SKILL_SOURCES: tuple[SkillSource, ...] = (RegistrySkillSource(), GitSkillSource())
+SKILL_SOURCE_ORDER: tuple[str, ...] = tuple(source.type for source in SKILL_SOURCES)
+
+
+def load_requested_skills() -> dict[str, list[Any]]:
+ """Read requested file into per-source buckets. Missing → empty (clean up all sources)."""
+ buckets: dict[str, list[Any]] = {skill_type: [] for skill_type in SKILL_SOURCE_ORDER}
+ path = SKILLS_ROOT / REQUESTED_FILE_NAME
+ if not path.is_file():
+ return buckets
+ try:
+ requested = RequestedSkillsFileContent.model_validate_json(path.read_text(encoding="utf-8"))
+ except OSError as e:
+ sys.exit(f"Could not read {REQUESTED_FILE_NAME}: {e}")
+ except ValidationError as e:
+ sys.exit(f"Invalid {REQUESTED_FILE_NAME}: {e}")
+
+ for item in requested.skills:
+ buckets[item.type].append(item.resolve())
+ return buckets
+
+
+def run_skill_download() -> None:
+ loaded = load_requested_skills()
+ state = load_state()
+ removed_by_source = {source.type: source.reconcile(loaded[source.type], state) for source in SKILL_SOURCES}
+
+ if not any(loaded.values()):
+ bits = [
+ f"Removed {len(removed_by_source[skill_type])} {skill_type} skill(s)"
+ for skill_type in SKILL_SOURCE_ORDER
+ if removed_by_source[skill_type]
+ ]
+ if bits:
+ print(f"{'; '.join(bits)}. (requested empty.)")
+ save_state(state)
+ _delete_requested_file()
+ return
+
+ counts_by_source = {
+ source.type: (
+ source.download(loaded[source.type], state) if loaded[source.type] else (0, 0)
+ )
+ for source in SKILL_SOURCES
+ }
+ save_state(state)
+ _delete_requested_file()
+
+ suffix_parts = [
+ f"Removed {len(removed_by_source[skill_type])} stale {skill_type} skill(s)"
+ for skill_type in SKILL_SOURCE_ORDER
+ if removed_by_source[skill_type]
+ ]
+ suffix = f" {' '.join(suffix_parts)}." if suffix_parts else ""
+
+ failed_parts = [
+ f"{len(loaded[skill_type]) - sum(counts_by_source[skill_type])}/{len(loaded[skill_type])} {skill_type}"
+ for skill_type in SKILL_SOURCE_ORDER
+ if loaded[skill_type] and sum(counts_by_source[skill_type]) < len(loaded[skill_type])
+ ]
+ if failed_parts:
+ sys.exit(
+ f"Failed to install {' and '.join(failed_parts)} skill(s); see warnings above.{suffix}"
+ )
+
+ ensured = []
+ for skill_type in SKILL_SOURCE_ORDER:
+ if not loaded[skill_type]:
+ continue
+ installed, skipped = counts_by_source[skill_type]
+ ensured.append(f"{installed} installed/{skipped} skipped {skill_type}")
+ print(f"Ensured {' and '.join(ensured)} skill(s).{suffix}")
+
+
+def main() -> None:
+ run_skill_download()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/packages/trueforge-core/src/core/sandbox/skills/ISkillMounter.ts b/packages/trueforge-core/src/core/sandbox/skills/ISkillMounter.ts
index 07fbd601e..e1225c658 100644
--- a/packages/trueforge-core/src/core/sandbox/skills/ISkillMounter.ts
+++ b/packages/trueforge-core/src/core/sandbox/skills/ISkillMounter.ts
@@ -7,5 +7,5 @@ export interface ISkillMounter {
// Renders the <skills> section; adds nothing when empty, so InstructionBuilder drops the section.
instruction(builder: InstructionBuilder, paths: { skillsDir: string }): void;
// The command (+ env, timeout) that installs these skills; Sandbox folds it into its init exec.
- getSandboxInit(paths: { skillsDir: string; gitDownloaderPath: string }): SandboxInit;
+ getSandboxInit(paths: { skillsDir: string; skillDownloaderPath: string }): SandboxInit;
}
diff --git a/packages/trueforge-core/src/core/sandbox/skills/SkillMounter.ts b/packages/trueforge-core/src/core/sandbox/skills/SkillMounter.ts
index 6037b5be3..415ebbef7 100644
--- a/packages/trueforge-core/src/core/sandbox/skills/SkillMounter.ts
+++ b/packages/trueforge-core/src/core/sandbox/skills/SkillMounter.ts
@@ -5,34 +5,84 @@ import { sandboxScripts } from '../sandboxScripts.gen';
import { SKILLS_PREAMBLE, getSkillPath, renderSkillPromptBody } from './constants';
import type { ISkillMounter } from './ISkillMounter';
-// A git-sourced skill materialized by a sparse clone in the sandbox (git_downloader.py). Git skills
-// never preload — their SKILL.md is read from disk at runtime (only name/description/path are
-// advertised in the prompt). Wire fields match the agent_spec git mount (`url`/`path`/`name`/`ref`);
-// the downloader resolves `ref` inside the sandbox before fetching.
+/** Git skill: sparse-cloned in the sandbox by skill_downloader.py (never preloaded). */
export interface GitSkill {
+ readonly type: 'git';
readonly name: string;
readonly description: string;
// Canonical https clone URL of the github.com/gitlab.com repo (host-validated by the caller).
readonly url: string;
// Subdirectory within the repo that holds the skill (empty = repo root).
readonly path: string;
- // Branch, tag, or full object id — resolved and fetched by git_downloader.py in the sandbox.
+ // Branch, tag, or full object id — resolved and fetched by skill_downloader.py in the sandbox.
readonly ref: string;
}
-// Serializable descriptor handed to git_downloader.py (base64 JSON); keys match its GitSkill model.
-interface GitSkillSpec {
- name: string;
- url: string;
- path: string;
- ref: string;
+/** Registry skill: sandbox dir uses artifact `name`; AgentSpec stores `fqn`. */
+interface RegistrySkill {
+ readonly type: 'registry';
+ readonly name: string;
+ readonly description: string;
+ // Version FQN from AgentSpec (`agent-skill:tenant/ml_repo/name:N`).
+ readonly fqn: string;
+ // When true, inline `skillMdContent` into the prompt instead of reading SKILL.md from disk.
+ readonly preload: boolean;
+ // SKILL.md body when `preload` is true; otherwise null.
+ readonly skillMdContent: string | null;
+ // Short-lived URL the sandbox uses to download the skill tarball.
+ readonly presignedUrl: string;
+}
+
+export type Skill = GitSkill | RegistrySkill;
+
+export const REQUESTED_SKILLS_FILE_NAME = '.tfy-requested-skills.json';
+
+type RequestedGitSkill = Pick<GitSkill, 'type' | 'name' | 'url' | 'path' | 'ref'>;
+
+type RequestedRegistrySkill = Pick<RegistrySkill, 'type' | 'name' | 'fqn'> & {
+ // Host↔sandbox wire: JSON/Python use snake_case; RegistrySkill stays camelCase.
+ presigned_url: RegistrySkill['presignedUrl'];
+};
+
+type RequestedSkill = RequestedGitSkill | RequestedRegistrySkill;
+
+function toRequestedSkill(skill: Skill): RequestedSkill {
+ switch (skill.type) {
+ case 'registry': {
+ const { name, fqn, presignedUrl: presigned_url } = skill;
+ // Rename at upload: downloader reads `presigned_url`, never `presignedUrl`.
+ return { type: 'registry', name, fqn, presigned_url };
+ }
+ case 'git': {
+ const { name, url, path, ref } = skill;
+ return { type: 'git', name, url, path, ref };
+ }
+ default: {
+ const _exhaustive: never = skill;
+ throw new Error(`unexpected skill: ${JSON.stringify(_exhaustive)}`);
+ }
+ }
+}
+
+/** Inline SKILL.md for the prompt when the registry skill is preloaded; git never preloads. */
+function resolvePreloadContent(skill: Skill): string | null {
+ switch (skill.type) {
+ case 'registry':
+ return skill.preload ? skill.skillMdContent : null;
+ case 'git':
+ return null;
+ default: {
+ const _exhaustive: never = skill;
+ throw new Error(`unexpected skill: ${JSON.stringify(_exhaustive)}`);
+ }
+ }
}
export class SkillMounter implements ISkillMounter {
- private readonly skills: readonly GitSkill[];
+ private readonly skills: readonly Skill[];
- constructor(skills: readonly GitSkill[]) {
- this.skills = skills;
+ constructor(input: { skills?: readonly Skill[] } = {}) {
+ this.skills = input.skills ?? [];
}
instruction(builder: InstructionBuilder, paths: { skillsDir: string }): void {
@@ -41,35 +91,45 @@ export class SkillMounter implements ISkillMounter {
}
builder.addContent(SKILLS_PREAMBLE);
for (const skill of this.skills) {
- builder.addSection(
- 'skill',
- renderSkillPromptBody({
- path: getSkillPath({ skillsDir: paths.skillsDir, skillName: skill.name }),
- name: skill.name,
- description: skill.description,
- preloadContent: null,
- }),
- );
+ this.#addPromptSkill(builder, paths.skillsDir, {
+ name: skill.name,
+ description: skill.description,
+ preloadContent: resolvePreloadContent(skill),
+ });
}
}
- getSandboxInit(paths: { skillsDir: string; gitDownloaderPath: string }): SandboxInit {
- // An empty desired set is also the source-neutral cleanup path for a reused sandbox.
- const specs: GitSkillSpec[] = this.skills.map(skill => ({
- name: skill.name,
- url: skill.url,
- path: skill.path,
- ref: skill.ref,
- }));
- const gitSkillsB64 = Buffer.from(JSON.stringify(specs)).toString('base64');
+ getSandboxInit(paths: { skillsDir: string; skillDownloaderPath: string }): SandboxInit {
+ // Always upload (including empty) so that existing skills are cleaned up.
return {
command: buildWriteAndRunScriptCommand({
- scriptPath: paths.gitDownloaderPath,
- // Bundled at build time (sandboxScripts.gen.ts) so the packaged library has no loose files.
- scriptContent: sandboxScripts.gitDownloader,
+ scriptPath: paths.skillDownloaderPath,
+ scriptContent: sandboxScripts.skillDownloader,
}),
- env: { AGENT_GIT_SKILLS: gitSkillsB64, TFY_SKILLS_DIR: paths.skillsDir },
+ env: { TFY_SKILLS_DIR: paths.skillsDir },
timeoutSeconds: SKILL_DOWNLOAD_TIMEOUT_SECONDS,
+ uploads: [
+ {
+ remotePath: `${paths.skillsDir}/${REQUESTED_SKILLS_FILE_NAME}`,
+ content: Buffer.from(JSON.stringify({ skills: this.skills.map(toRequestedSkill) }), 'utf-8'),
+ },
+ ],
};
}
+
+ #addPromptSkill(
+ builder: InstructionBuilder,
+ skillsDir: string,
+ skill: { name: string; description: string; preloadContent: string | null },
+ ): void {
+ builder.addSection(
+ 'skill',
+ renderSkillPromptBody({
+ path: getSkillPath({ skillsDir, skillName: skill.name }),
+ name: skill.name,
+ description: skill.description,
+ preloadContent: skill.preloadContent,
+ }),
+ );
+ }
}
diff --git a/packages/trueforge-core/src/core/sandbox/skills/index.ts b/packages/trueforge-core/src/core/sandbox/skills/index.ts
index 23636c14b..9538258fb 100644
--- a/packages/trueforge-core/src/core/sandbox/skills/index.ts
+++ b/packages/trueforge-core/src/core/sandbox/skills/index.ts
@@ -1,4 +1,4 @@
export { SKILLS_PREAMBLE, getSkillPath, renderSkillPromptBody } from './constants';
export type { ISkillMounter } from './ISkillMounter';
export { SkillMounter } from './SkillMounter';
-export type { GitSkill } from './SkillMounter';
+export type { GitSkill, Skill } from './SkillMounter';
diff --git a/packages/trueforge-core/tests/core/harnessMocks.ts b/packages/trueforge-core/tests/core/harnessMocks.ts
index 22d110b5f..81bfccf03 100644
--- a/packages/trueforge-core/tests/core/harnessMocks.ts
+++ b/packages/trueforge-core/tests/core/harnessMocks.ts
@@ -67,7 +67,7 @@ export function makeStubPublicSandbox(): Sandbox {
getGitCredentialsPath: () => '/tmp/.git-credentials',
getFileUploadsDir: () => '/tmp/uploads',
getSkillsDir: () => '/opt/tfy/skills',
- getGitDownloaderPath: () => '/opt/tfy/git_downloader.py',
+ getSkillDownloaderPath: () => '/opt/tfy/skill_downloader.py',
downloadFile: jest.fn(),
uploadFile: jest.fn(),
createCodeModeTransport: jest.fn(),
diff --git a/packages/trueforge-core/tests/core/sandbox/Sandbox.ids.test.ts b/packages/trueforge-core/tests/core/sandbox/Sandbox.ids.test.ts
index 27673b0ae..c9273ce30 100644
--- a/packages/trueforge-core/tests/core/sandbox/Sandbox.ids.test.ts
+++ b/packages/trueforge-core/tests/core/sandbox/Sandbox.ids.test.ts
@@ -21,7 +21,7 @@ function makeProvider(
getGitCredentialsPath: sandboxId => `${sandboxId}/.git-credentials`,
getFileUploadsDir: sandboxId => `${sandboxId}/uploads`,
getSkillsDir: sandboxId => `${sandboxId}/skills`,
- getGitDownloaderPath: sandboxId => `${sandboxId}/git_downloader.py`,
+ getSkillDownloaderPath: sandboxId => `${sandboxId}/skill_downloader.py`,
downloadFile: jest.fn(),
uploadFile: jest.fn().mockResolvedValue(undefined),
createCodeModeTransport: jest.fn(),
diff --git a/packages/trueforge-core/tests/core/sandbox/Sandbox.paths.test.ts b/packages/trueforge-core/tests/core/sandbox/Sandbox.paths.test.ts
index 7184725af..1449edd3c 100644
--- a/packages/trueforge-core/tests/core/sandbox/Sandbox.paths.test.ts
+++ b/packages/trueforge-core/tests/core/sandbox/Sandbox.paths.test.ts
@@ -2,6 +2,7 @@ import { InstructionBuilder } from '../../../src/core/InstructionBuilder';
import type { CodeModeTransport } from '../../../src/core/sandbox/codeMode/CodeModeTransport';
import type { ExecResult, SandboxExecParams, SandboxProvider } from '../../../src/core/sandbox/provider/Provider';
import { SANDBOX_EXEC_TOOL_NAME, Sandbox } from '../../../src/core/sandbox/Sandbox';
+import { REQUESTED_SKILLS_FILE_NAME, SkillMounter, type GitSkill } from '../../../src/core/sandbox/skills/SkillMounter';
import { NOOP_AGENT_TRACING } from '../../../src/core/tracing/NoopAgentTracing';
import { makeMockIMCPServer, makeSilentLogger } from '../harnessMocks';
@@ -21,7 +22,7 @@ function makeProvider(overrides: Partial<SandboxProvider> = {}): SandboxProvider
getGitCredentialsPath: () => '/prov/.git-credentials',
getFileUploadsDir: () => '/prov/uploads',
getSkillsDir: () => '/prov/skills',
- getGitDownloaderPath: () => '/prov/git_downloader.py',
+ getSkillDownloaderPath: () => '/prov/skill_downloader.py',
downloadFile: jest.fn(),
uploadFile: jest.fn().mockResolvedValue(undefined),
createCodeModeTransport: jest.fn(),
@@ -29,10 +30,14 @@ function makeProvider(overrides: Partial<SandboxProvider> = {}): SandboxProvider
};
}
-function makeSandbox(provider: SandboxProvider, options: { existingSandboxId?: string } = {}): Sandbox {
+function makeSandbox(
+ provider: SandboxProvider,
+ options: { existingSandboxId?: string; skillMounter?: SkillMounter } = {},
+): Sandbox {
return new Sandbox({
provider,
existingSandboxId: options.existingSandboxId,
+ skillMounter: options.skillMounter,
blockDestructiveToolsInCodeMode: true,
mcpRequestTimeoutMs: 60_000,
mcpConnectTimeoutMs: 5_000,
@@ -68,6 +73,75 @@ describe('Sandbox provider-owned paths', () => {
});
});
+ it('uploads skill-init files from the mounter before the skill init exec', async () => {
+ const events: string[] = [];
+ const uploadFile = jest.fn().mockImplementation(async (args: { remotePath: string }) => {
+ if (args.remotePath === `/prov/skills/${REQUESTED_SKILLS_FILE_NAME}`) {
+ events.push('skill-upload');
+ }
+ });
+ const exec = jest.fn().mockImplementation((params: SandboxExecParams): Promise<ExecResult> => {
+ if (params.env?.['TFY_SKILLS_DIR'] === '/prov/skills') {
+ events.push('skill-init-exec');
+ }
+ return readyExec();
+ });
+ const gitSkill = {
+ type: 'git',
+ name: 'echo',
+ description: 'Echo',
+ url: 'https://github.com/acme/skills.git',
+ path: '',
+ ref: 'a'.repeat(40),
+ } satisfies GitSkill;
+ const sandbox = makeSandbox(makeProvider({ uploadFile, exec }), {
+ skillMounter: new SkillMounter({ skills: [gitSkill] }),
+ });
+
+ await sandbox.uploadUserFile({
+ fileName: 'notes.txt',
+ content: Buffer.from('hi'),
+ mime: 'text/plain',
+ });
+
+ const skillsUpload = uploadFile.mock.calls
+ .map((call: unknown[]) => call[0] as { remotePath: string; content: Buffer; sandboxId: string })
+ .find(args => args.remotePath === `/prov/skills/${REQUESTED_SKILLS_FILE_NAME}`);
+ expect(skillsUpload).toBeDefined();
+ if (skillsUpload === undefined) {
+ return;
+ }
+ expect(skillsUpload).toEqual({
+ sandboxId: 'raw-1',
+ remotePath: `/prov/skills/${REQUESTED_SKILLS_FILE_NAME}`,
+ content: expect.any(Buffer),
+ });
+ expect(JSON.parse(skillsUpload.content.toString('utf-8'))).toEqual({
+ skills: [
+ {
+ type: 'git',
+ name: 'echo',
+ url: gitSkill.url,
+ path: '',
+ ref: gitSkill.ref,
+ },
+ ],
+ });
+ expect(events.indexOf('skill-upload')).toBeGreaterThanOrEqual(0);
+ expect(events.indexOf('skill-init-exec')).toBeGreaterThan(events.indexOf('skill-upload'));
+ expect(
+ exec.mock.calls.some((call: unknown[]) => {
+ const params = call[0] as SandboxExecParams;
+ return (
+ params.env?.['TFY_SKILLS_DIR'] === '/prov/skills' &&
+ params.timeoutSeconds === 180 &&
+ typeof params.command === 'string' &&
+ params.command.includes('/prov/skill_downloader.py')
+ );
+ }),
+ ).toBe(true);
+ });
+
it('does not rewrite a caller PATH (providers own PATH)', async () => {
const execCalls: SandboxExecParams[] = [];
const transport: CodeModeTransport = {
diff --git a/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts b/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts
index 8cdf834ef..4599e05d7 100644
--- a/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts
+++ b/packages/trueforge-core/tests/core/sandbox/provider/tfyLayout.test.ts
@@ -17,7 +17,7 @@ describe('TFYSandboxProvider layout', () => {
provider.getGitCredentialsPath(),
provider.getFileUploadsDir(),
provider.getSkillsDir(),
- provider.getGitDownloaderPath(),
+ provider.getSkillDownloaderPath(),
];
for (const path of layout) {
expect(path.startsWith('/')).toBe(false);
diff --git a/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts b/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts
index c7b90c1b3..1418e6b46 100644
--- a/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts
+++ b/packages/trueforge-core/tests/core/sandbox/sandboxBridgeTimeout.test.ts
@@ -30,7 +30,7 @@ function makeSandbox(options: {
getGitCredentialsPath: () => '/tmp/.git-credentials',
getFileUploadsDir: () => '/tmp/uploads',
getSkillsDir: () => '/opt/tfy/skills',
- getGitDownloaderPath: () => '/opt/tfy/git_downloader.py',
+ getSkillDownloaderPath: () => '/opt/tfy/skill_downloader.py',
downloadFile: jest.fn(),
uploadFile: jest.fn(),
createCodeModeTransport: () => {
diff --git a/packages/trueforge-core/tests/core/sandbox/skills/skillMounter.test.ts b/packages/trueforge-core/tests/core/sandbox/skills/skillMounter.test.ts
index aa41fc036..ccde02e98 100644
--- a/packages/trueforge-core/tests/core/sandbox/skills/skillMounter.test.ts
+++ b/packages/trueforge-core/tests/core/sandbox/skills/skillMounter.test.ts
@@ -1,61 +1,90 @@
/**
- * Public git-based SkillMounter: prompt rendering and the declarative sandbox
- * init it hands to Sandbox. (Host-specific mounters, e.g. the gateway's
- * TrueFoundry registry mounter, are tested in their own repo.)
+ * SkillMounter: prompt rendering and the declarative sandbox init it hands to Sandbox.
*/
import { InstructionBuilder } from '../../../../src/core/InstructionBuilder';
import type { ISkillMounter } from '../../../../src/core/sandbox/skills/ISkillMounter';
-import { SkillMounter } from '../../../../src/core/sandbox/skills/SkillMounter';
+import {
+ REQUESTED_SKILLS_FILE_NAME,
+ SkillMounter,
+ type GitSkill,
+ type Skill,
+} from '../../../../src/core/sandbox/skills/SkillMounter';
const GIT_SKILL = {
+ type: 'git',
name: 'git-skill',
description: 'From git',
url: 'https://github.com/acme/skills.git',
path: '',
ref: 'a'.repeat(40),
-};
+} satisfies GitSkill;
+
+const REGISTRY_SKILL = {
+ type: 'registry',
+ name: 'echo',
+ description: 'Echo',
+ fqn: 'agent-skill:acme/team/echo:1',
+ preload: false,
+ skillMdContent: null,
+ presignedUrl: 'https://example.com/echo.tar',
+} satisfies Skill;
-const PROVIDER_PATHS = {
+const PATHS = {
skillsDir: '/custom/skills',
- gitDownloaderPath: '/custom/git_downloader.py',
+ skillDownloaderPath: '/custom/skill_downloader.py',
};
-// Renders a mounter's <skills> section the same way Sandbox does (empty section => '').
function renderSkills(mounter: ISkillMounter): string {
const builder = new InstructionBuilder('skills');
- mounter.instruction(builder, { skillsDir: PROVIDER_PATHS.skillsDir });
+ mounter.instruction(builder, { skillsDir: PATHS.skillsDir });
return builder.build();
}
-describe('SkillMounter (public, git-only)', () => {
- it('embeds the injected git downloader and TFY_SKILLS_DIR', () => {
- const init = new SkillMounter([GIT_SKILL]).getSandboxInit(PROVIDER_PATHS);
+function readRequestedFile(mounter: SkillMounter): unknown {
+ const { uploads } = mounter.getSandboxInit(PATHS);
+ expect(uploads).toHaveLength(1);
+ const content = uploads[0]?.content.toString('utf-8');
+ expect(typeof content).toBe('string');
+ return JSON.parse(content ?? '');
+}
+
+describe('SkillMounter', () => {
+ it('uploads git skills in the requested file and points the init command at the downloader', () => {
+ const mounter = new SkillMounter({ skills: [GIT_SKILL] });
+ const init = mounter.getSandboxInit(PATHS);
- expect(init.command).toContain('/custom/git_downloader.py');
- expect(init.command).not.toContain('/opt/tfy/git_downloader.py');
- expect(init.env?.['AGENT_GIT_SKILLS']).toBeDefined();
- expect(init.env?.['TFY_SKILLS_DIR']).toBe('/custom/skills');
- expect(init.env?.['AGENT_SKILL_VERSION_FQNS']).toBeUndefined();
- expect(init.env?.['TFY_API_KEY']).toBeUndefined();
+ expect(init.command).toContain(PATHS.skillDownloaderPath);
+ expect(init.env?.['TFY_SKILLS_DIR']).toBe(PATHS.skillsDir);
expect(init.timeoutSeconds).toBe(180);
+ expect(init.uploads[0]?.remotePath).toBe(`${PATHS.skillsDir}/${REQUESTED_SKILLS_FILE_NAME}`);
+ expect(readRequestedFile(mounter)).toEqual({
+ skills: [{ type: 'git', name: 'git-skill', url: GIT_SKILL.url, path: '', ref: GIT_SKILL.ref }],
+ });
});
- it('renders a <skills> section with a block per skill using the injected skills dir', () => {
- const rendered = renderSkills(new SkillMounter([GIT_SKILL]));
-
+ it('renders skill prompt blocks under the injected skills dir', () => {
+ const rendered = renderSkills(new SkillMounter({ skills: [GIT_SKILL] }));
expect(rendered).toContain('<skills>');
- expect(rendered).toContain('git-skill');
- expect(rendered).toContain('/custom/skills/git-skill');
- expect(rendered).not.toContain('/opt/tfy/skills');
+ expect(rendered).toContain(`${PATHS.skillsDir}/git-skill`);
});
- it('uses an empty desired set for cleanup on a reused sandbox and renders nothing', () => {
- const mounter = new SkillMounter([]);
- const init = mounter.getSandboxInit(PROVIDER_PATHS);
-
+ it('uploads an empty requested file so that existing skills are cleaned up', () => {
+ const mounter = new SkillMounter({});
expect(renderSkills(mounter)).toBe('');
- expect(init.command).toContain('/custom/git_downloader.py');
- expect(init.env?.['TFY_SKILLS_DIR']).toBe('/custom/skills');
- expect(JSON.parse(Buffer.from(init.env?.['AGENT_GIT_SKILLS'] ?? '', 'base64').toString())).toEqual([]);
+ expect(readRequestedFile(mounter)).toEqual({ skills: [] });
+ });
+
+ it('writes git and registry entries into one requested file in caller order', () => {
+ expect(readRequestedFile(new SkillMounter({ skills: [GIT_SKILL, REGISTRY_SKILL] }))).toEqual({
+ skills: [
+ { type: 'git', name: 'git-skill', url: GIT_SKILL.url, path: '', ref: GIT_SKILL.ref },
+ {
+ type: 'registry',
+ name: 'echo',
+ fqn: 'agent-skill:acme/team/echo:1',
+ presigned_url: 'https://example.com/echo.tar',
+ },
+ ],
+ });
});
});
diff --git a/packages/trueforge/src/apis/turns.ts b/packages/trueforge/src/apis/turns.ts
index 53599eea3..d17931b06 100644
--- a/packages/trueforge/src/apis/turns.ts
+++ b/packages/trueforge/src/apis/turns.ts
@@ -232,7 +232,7 @@ function createTurnResolver(deps: {
return buildTurnSandbox({
provider,
logger,
- gitSkills,
+ skills: gitSkills,
fileDownloadEnabled: spec.config.sandbox.file_downloads,
existingSandboxId: carriedSandboxId,
tracing,
diff --git a/packages/trueforge/src/runtime/sessionResources.ts b/packages/trueforge/src/runtime/sessionResources.ts
index b82dd7ec7..c33550a42 100644
--- a/packages/trueforge/src/runtime/sessionResources.ts
+++ b/packages/trueforge/src/runtime/sessionResources.ts
@@ -8,6 +8,7 @@ import {
type ModelParams,
type RemoteMcpHeaders,
type SandboxProvider,
+ type Skill,
type VercelAIProviderConfig,
} from '@truefoundry/trueforge-core/core';
import { HTTPException } from 'hono/http-exception';
@@ -154,6 +155,7 @@ export async function resolveGitSkills({
});
}
resolved.push({
+ type: 'git',
name: record.manifest.name,
description: record.manifest.description,
url: record.manifest.url,
@@ -216,17 +218,17 @@ export async function resolveSandboxProvider({
}
/**
- * Builds a Sandbox for one turn from a resolved provider and git mounts.
+ * Builds a Sandbox for one turn from a resolved provider and skill mounts.
*/
export function buildTurnSandbox(input: {
provider: SandboxProvider;
logger: Logger;
- gitSkills: readonly GitSkill[];
+ skills?: readonly Skill[];
fileDownloadEnabled: boolean;
existingSandboxId?: string | undefined;
tracing: AgentTracing;
}): Sandbox {
- const skillMounter = input.gitSkills.length > 0 ? new SkillMounter([...input.gitSkills]) : undefined;
+ // Empty mounter still uploads requested-skills file so existing skills are cleaned up.
return new Sandbox({
provider: input.provider,
existingSandboxId: input.existingSandboxId,
@@ -234,7 +236,7 @@ export function buildTurnSandbox(input: {
blockDestructiveToolsInCodeMode: true,
mcpRequestTimeoutMs: configuration.MCP_REQUEST_TIMEOUT_MS,
mcpConnectTimeoutMs: configuration.MCP_CONNECT_TIMEOUT_MS,
- ...(skillMounter ? { skillMounter } : {}),
+ skillMounter: new SkillMounter({ skills: input.skills ?? [] }),
tracing: input.tracing,
logger: input.logger,
});
diff --git a/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts b/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts
index 43c7c7d4a..d03f8cada 100644
--- a/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts
+++ b/packages/trueforge/src/sandbox/local/provider/LocalSandboxProvider.ts
@@ -55,7 +55,7 @@ const DEFAULT_FILE_MAX_BYTES = 10 * 1024 * 1024;
const SUPPORT_PROBE_TIMEOUT_MS = 5_000;
const VENV_CREATE_TIMEOUT_MS = 60_000;
const VENV_PIP_TIMEOUT_MS = 120_000;
-/** Same pin as the Daytona image / git_downloader.py PEP 723 header. */
+/** Same pin as the Daytona image / skill_downloader.py PEP 723 header. */
const VENV_PYDANTIC_PIN = 'pydantic>=2.0.0,<3.0.0';
/** Command names resolved via `command -v` (PATH from sandbox policy). */
@@ -655,7 +655,7 @@ export class LocalSandboxProvider implements SandboxProvider {
// Cwd-relative: SRT cwd is the sandbox root. Init mkdir/ln and the agent prompt must not use
// host-absolute paths — those contain spaces under macOS Application Support and lose quoting.
- // uploads, skills, tool-results, git_downloader.py, .git-credentials, .venv
+ // uploads, skills, tool-results, skill_downloader.py, .git-credentials, .venv
// mcp-client/mcp_client.py
getToolResultDumpDir(): string {
return 'tool-results';
@@ -673,8 +673,8 @@ export class LocalSandboxProvider implements SandboxProvider {
return 'skills';
}
- getGitDownloaderPath(): string {
- return 'git_downloader.py';
+ getSkillDownloaderPath(): string {
+ return 'skill_downloader.py';
}
async downloadFile(params: { sandboxId: string; path: string }): Promise<Buffer> {
diff --git a/packages/trueforge/tests/unit/sandbox/local/provider/missingRoot.test.ts b/packages/trueforge/tests/unit/sandbox/local/provider/missingRoot.test.ts
index 8b96f249d..26d1c136a 100644
--- a/packages/trueforge/tests/unit/sandbox/local/provider/missingRoot.test.ts
+++ b/packages/trueforge/tests/unit/sandbox/local/provider/missingRoot.test.ts
@@ -81,7 +81,7 @@ describeUnix('LocalSandboxProvider missing root', () => {
expect(provider.getGitCredentialsPath()).toBe('.git-credentials');
expect(provider.getFileUploadsDir()).toBe('uploads');
expect(provider.getSkillsDir()).toBe('skills');
- expect(provider.getGitDownloaderPath()).toBe('git_downloader.py');
+ expect(provider.getSkillDownloaderPath()).toBe('skill_downloader.py');
const install = provider.createCodeModeTransport().getClientInstall({
sandboxId: join(sandboxRootPathParent, 'x'),
});