The open-source agent harness - the runtime layer that turns an LLM into a working agent.
Extract skill mounting into a SkillMounter module. (#3)
Srajan Asthanafa00af91a396
diff --git a/packages/harness/package.json b/packages/harness/package.jsonindex 686b859ea..e08dfb801 100644--- a/packages/harness/package.json+++ b/packages/harness/package.json@@ -1,6 +1,6 @@{"name": "@truefoundry/utils",- "version": "0.1.3",+ "version": "0.1.4","description": "TrueFoundry agent core and session library","author": "Truefoundry","license": "UNLICENSED",diff --git a/packages/harness/scripts/generate-sandbox-scripts.mjs b/packages/harness/scripts/generate-sandbox-scripts.mjsindex a94f06e66..a276ab440 100644--- a/packages/harness/scripts/generate-sandbox-scripts.mjs+++ b/packages/harness/scripts/generate-sandbox-scripts.mjs@@ -18,7 +18,7 @@ const read = f => readFileSync(join(scriptsDir, f), 'utf-8');const out = `// AUTO-GENERATED by scripts/generate-sandbox-scripts.mjs — do not edit.export const sandboxScripts = {mcpClient: ${JSON.stringify(read('mcp_client.py'))},- skillDownloader: ${JSON.stringify(read('skill_downloader.py'))},+ gitDownloader: ${JSON.stringify(read('git_downloader.py'))},} as const;`;writeFileSync(join(root, 'src/core/sandbox/sandboxScripts.gen.ts'), out);diff --git a/packages/harness/src/core/index.ts b/packages/harness/src/core/index.tsindex f05eec8e6..ed169b459 100644--- a/packages/harness/src/core/index.ts+++ b/packages/harness/src/core/index.ts@@ -140,8 +140,13 @@ export {export type { CreateSandboxProviderInput, SandboxProviderSettings } from './sandbox/provider/createSandboxProvider';export { DaytonaSandboxProvider } from './sandbox/provider/DaytonaProvider';export type { DaytonaSandboxSettings } from './sandbox/provider/DaytonaProvider';-export type { SandboxProvider } from './sandbox/provider/Provider';+export type { SandboxExecParams, SandboxInit, SandboxProvider } from './sandbox/provider/Provider';export { TFYSandboxProvider } from './sandbox/provider/TFYSandboxProvider';-export { Sandbox } from './sandbox/Sandbox';-export type { MountedSkill, SandboxInfo } from './sandbox/Sandbox';+export { SKILL_DOWNLOAD_TIMEOUT_SECONDS, Sandbox, buildWriteAndRunScriptCommand } from './sandbox/Sandbox';+export type { SandboxInfo } from './sandbox/Sandbox';export { SandboxError } from './sandbox/SandboxErrors';++// Skills: the ISkillMounter seam lets hosts plug in their own skill sources+export { InstructionBuilder } from './InstructionBuilder';+export { SKILLS_PREAMBLE, SkillMounter, getSkillPath, renderSkillPromptBody } from './sandbox/skills';+export type { GitSkill, ISkillMounter } from './sandbox/skills';diff --git a/packages/harness/src/core/sandbox/Sandbox.ts b/packages/harness/src/core/sandbox/Sandbox.tsindex a98abd880..b48942496 100644--- a/packages/harness/src/core/sandbox/Sandbox.ts+++ b/packages/harness/src/core/sandbox/Sandbox.ts@@ -5,7 +5,7 @@ import { z } from 'zod';import { InstructionBuilder } from '../InstructionBuilder';import { estimateTokensForString } from '../llm/usage';import type { MappedMCPTool } from '../mcp/convertMCPServers';-import { type CallToolResponse, type IToolSet, toolResultResponse } from '../mcp/IMCPServer';+import { toolResultResponse, type CallToolResponse, type IToolSet } from '../mcp/IMCPServer';import { defineTool, LocalToolMCP } from '../mcp/LocalToolMCP';import {DEFERRED_TOOLS_SERVER_ID,@@ -16,15 +16,33 @@ import {import type { AgentTracing } from '../tracing/AgentTracing';import { extractErrorLogFields } from '../util/errorLogFields';import { SANDBOX_FILE_UPLOADS_DIR, SANDBOX_NATS_WS_PORT } from './constants';-import { type SandboxProvider, ensureExecSuccess } from './provider/Provider';+import { ensureExecSuccess, shellEscape, type SandboxProvider } from './provider/Provider';import { validateNoPathTraversal, validateSandboxOwnedByTenant } from './SandboxErrors';import { SandboxNatsBridge } from './SandboxNatsBridge';import { sandboxScripts } from './sandboxScripts.gen';+// Import submodules, not the ./skills barrel, to avoid a cycle (the mounters import from Sandbox).+import { SKILLS_DIR } from './skills/constants';+import type { ISkillMounter } from './skills/ISkillMounter';export interface SandboxInfo {sandbox_id: string;}+// Downloader/setup scripts can run longer than a normal exec.+export const SKILL_DOWNLOAD_TIMEOUT_SECONDS = 180;++// Write a script (base64-encoded, never interpolated raw) to a path and run it. Skill mounters reuse it.+export function buildWriteAndRunScriptCommand(params: { scriptPath: string; scriptContent: string }): string {+ const b64 = Buffer.from(params.scriptContent, 'utf-8').toString('base64');+ return [+ `mkdir -p "$(dirname ${params.scriptPath})"`,+ `rm -f ${params.scriptPath}`,+ `echo ${shellEscape(b64)} | base64 -d > ${params.scriptPath}`,+ `chmod a-w ${params.scriptPath}`,+ `python3 ${params.scriptPath}`,+ ].join(' && ');+}+/** Write or clear the git credential-store file at an absolute path (no global git config mutation). */function buildSyncGitCredentialsCommand(credentialsContent: string | null, credentialsPath: string): string {if (credentialsContent === null) {@@ -51,13 +69,6 @@ function buildGitCredentialHelperEnv(credentialsPath: string): Record<string, st};}-export interface MountedSkill {- fqn: string;- name: string;- description: string;- skillMdContent: string | null;-}-export interface SandboxStoredFile {filePath: string;sandboxCreated?: SandboxInfo | undefined;@@ -66,7 +77,7 @@ export interface SandboxStoredFile {export interface SandboxOptions {provider: SandboxProvider;existingSandboxId?: string | undefined;- mountedSkills?: readonly MountedSkill[] | undefined;+ skillMounter?: ISkillMounter | undefined;fileDownloadEnabled?: boolean | undefined;/** Pre-resolved credential-store file content (null = clear / no git auth). */resolvedGitCredentialsContent?: string | null | undefined;@@ -76,7 +87,6 @@ export interface SandboxOptions {*/blockDestructiveToolsInCodeMode: true;execExtraEnv?: Readonly<Record<string, string>> | undefined;- skillInitEnv?: Readonly<Record<string, string>> | undefined;tracing: AgentTracing;logger: Logger;}@@ -95,22 +105,6 @@ const SANDBOX_FILE_UPLOADS_TAG = 'sandbox-file-uploads';const MCP_CLIENT_DIR = '/opt/tfy/mcp-client';const MCP_CLIENT_PATH = `${MCP_CLIENT_DIR}/mcp_client.py`;const MCP_CLIENT_BIN_SYMLINK = '/usr/local/bin/mcp-client';-const SKILL_DOWNLOADER_PATH = '/opt/tfy/skill_downloader.py';-const SKILLS_DIR = '/opt/tfy/skills';--function getSkillPath(skillName: string): string {- return `${SKILLS_DIR}/${skillName}`;-}--const SKILLS_PREAMBLE = [- '1. The skill is present in the `path` directory.',- '2. SKILL.md file within the directory is the primary file, there can be other files as well.',- '3. Use `description` to judge whether the Agent needs to use a skill.',- '4. To use a skill:',- '4.a. If the `SKILL.md content` is inlined, the Agent must not read `path/SKILL.md`.',- '4.b. Otherwise the Agent must read `path/SKILL.md` first.',- '5. The Agent must avoid listing the `path` unless there are references in `SKILL.md` and there is reason to do so given the current conversation.',-].join('\n');// Per-server config sent to mcp_client.py via TFY_MCP_SERVERS.interface SandboxMcpServerConfig {@@ -163,16 +157,6 @@ function injectMCPClientEnv(params: {};}-function buildSkillInitEnv(- skillInitEnv: Readonly<Record<string, string>> | undefined,- skillVersionFqnsCsv?: string,-): Record<string, string> {- return {- AGENT_SKILL_VERSION_FQNS: skillVersionFqnsCsv ?? '',- ...(skillInitEnv ?? {}),- };-}-const sandboxExecSchema = z.object({intent: z.string()@@ -214,10 +198,9 @@ export class Sandbox extends LocalToolMCP {private codeExecToolSets: readonly IToolSet[] = [];private readonly fileDownloadEnabled: boolean;private readonly execExtraEnv?: Readonly<Record<string, string>> | undefined;- private readonly skillInitEnv?: Readonly<Record<string, string>> | undefined;- private readonly mountedSkills: readonly MountedSkill[];+ private readonly skillMounter?: ISkillMounter | undefined;+ private cachedSkillsSection?: string | undefined;private readonly mcpClientScriptBase64: string;- private readonly skillDownloaderScriptBase64: string;private readonly logger: Logger;// Pre-resolved credential-store file content (null = clear / no git auth).private readonly resolvedGitCredentialsContent: string | null;@@ -237,15 +220,13 @@ export class Sandbox extends LocalToolMCP {this.provider = options.provider;this.existingSandboxId = options.existingSandboxId;this.tenantName = options.execExtraEnv?.['TFY_TENANT_NAME'] ?? '';- this.mountedSkills = options.mountedSkills ?? [];+ this.skillMounter = options.skillMounter;this.fileDownloadEnabled = options.fileDownloadEnabled ?? false;this.execExtraEnv = options.execExtraEnv;- this.skillInitEnv = options.skillInitEnv;// Scripts are internal to Sandbox: the upload paths, env contract, and prompt// text are all hardcoded here, so injecting different content was never a// real extension point. Consumers don't see or provide them.this.mcpClientScriptBase64 = Buffer.from(sandboxScripts.mcpClient, 'utf-8').toString('base64');- this.skillDownloaderScriptBase64 = Buffer.from(sandboxScripts.skillDownloader, 'utf-8').toString('base64');this.logger = options.logger.child({ module: 'Sandbox' });this.resolvedGitCredentialsContent = options.resolvedGitCredentialsContent ?? null;@@ -276,17 +257,17 @@ export class Sandbox extends LocalToolMCP {}estimateSkillsTokens(): number {- if (!this.mountedSkills.length) return 0;- let fullContent = `<skills>\n${SKILLS_PREAMBLE}\n\n`;- for (const skill of this.mountedSkills) {- const skillPath = getSkillPath(skill.name);- const body = skill.skillMdContent- ? `path: ${skillPath}\nSKILL.md content:\n${skill.skillMdContent}`- : `path: ${skillPath}\nname: ${skill.name}\ndescription: ${skill.description}`;- fullContent += `<skill>\n${body}\n</skill>\n\n`;+ return Math.round(estimateTokensForString(this.renderSkillsSection()));+ }++ // Single source for the <skills> section, so the prompt and the token estimate can't drift.+ private renderSkillsSection(): string {+ if (this.cachedSkillsSection === undefined) {+ const skills = new InstructionBuilder('skills');+ this.skillMounter?.instruction(skills);+ this.cachedSkillsSection = skills.build();}- fullContent += '</skills>';- return Math.round(estimateTokensForString(fullContent));+ return this.cachedSkillsSection;}buildInstruction(builder: InstructionBuilder): void {@@ -354,19 +335,7 @@ export class Sandbox extends LocalToolMCP {}private buildSkillsSection(builder: InstructionBuilder): void {- if (!this.mountedSkills.length) return;-- const skills = builder.beginSection('skills');-- skills.addContent(SKILLS_PREAMBLE);-- for (const skill of this.mountedSkills) {- const skillPath = getSkillPath(skill.name);- const content = skill.skillMdContent- ? `path: ${skillPath}\nSKILL.md content:\n${skill.skillMdContent}`- : `path: ${skillPath}\nname: ${skill.name}\ndescription: ${skill.description}`;- skills.addSection('skill', content);- }+ builder.addContent(this.renderSkillsSection());}private buildMCPSection(builder: InstructionBuilder): void {@@ -604,37 +573,38 @@ export class Sandbox extends LocalToolMCP {}private async initSandboxEnvironment(): Promise<void> {- const skillVersionFqnsCsv = this.mountedSkills.length ? this.mountedSkills.map(s => s.fqn).join(',') : undefined;const toolResultDumpDir = this.provider.getToolResultDumpDir(this.requiredSandboxInfo.sandbox_id);this.logger.info('Uploading MCP client script and preparing skills directory in sandbox');const initSteps = [- `mkdir -p ${MCP_CLIENT_DIR} ${Sandbox.FILE_UPLOADS_DIR} ${toolResultDumpDir}`,+ `mkdir -p ${MCP_CLIENT_DIR} ${Sandbox.FILE_UPLOADS_DIR} ${toolResultDumpDir} ${SKILLS_DIR}`,`rm -f ${MCP_CLIENT_PATH}`,`echo '${this.mcpClientScriptBase64}' | base64 -d > ${MCP_CLIENT_PATH}`,// Make executable + symlink onto PATH so `mcp-client` runs by name.`chmod 0555 ${MCP_CLIENT_PATH}`,`ln -sf ${MCP_CLIENT_PATH} ${MCP_CLIENT_BIN_SYMLINK}`,- `rm -f ${SKILL_DOWNLOADER_PATH}`,- `echo '${this.skillDownloaderScriptBase64}' | base64 -d > ${SKILL_DOWNLOADER_PATH}`,- `chmod a-w ${SKILL_DOWNLOADER_PATH}`,- `mkdir -p ${SKILLS_DIR}`,- `python3 ${SKILL_DOWNLOADER_PATH}`,];++ // Fold skill installation into this single init exec. The mounter returns a declarative+ // command + env + timeout; runs even when empty so its downloader can prune a reused sandbox.+ const skillInit = this.skillMounter?.getSandboxInit();+ if (skillInit) {+ initSteps.push(skillInit.command);+ }+const script = initSteps.join(' && ');const result = await this.provider.exec({sandboxId: this.requiredSandboxInfo.sandbox_id,command: script,- env: buildSkillInitEnv(this.skillInitEnv, skillVersionFqnsCsv),+ env: skillInit?.env,+ timeoutSeconds: skillInit?.timeoutSeconds,});ensureExecSuccess(result);- const fqnCount = this.mountedSkills.length;this.logger.info(- `Sandbox initialized: MCP client at ${MCP_CLIENT_PATH} (symlinked to ${MCP_CLIENT_BIN_SYMLINK}); ` +- `skill downloader at ${SKILL_DOWNLOADER_PATH}; skills dir ${SKILLS_DIR}` +- `; ran skill downloader with ${String(fqnCount)} FQN(s) input`,+ `Sandbox initialized: MCP client at ${MCP_CLIENT_PATH} (symlinked to ${MCP_CLIENT_BIN_SYMLINK}); skills dir ${SKILLS_DIR}`,);+await this.writeGitCredentials();}diff --git a/packages/harness/src/core/sandbox/provider/DaytonaProvider.ts b/packages/harness/src/core/sandbox/provider/DaytonaProvider.tsindex 6c03caeed..157ca6148 100644--- a/packages/harness/src/core/sandbox/provider/DaytonaProvider.ts+++ b/packages/harness/src/core/sandbox/provider/DaytonaProvider.ts@@ -11,7 +11,7 @@ import {SandboxNotAvailableError,SandboxPathIsDirectoryError,} from '../SandboxErrors';-import type { ExecResult, SandboxFileInfo, SandboxProvider } from './Provider';+import type { ExecResult, SandboxExecParams, SandboxFileInfo, SandboxProvider } from './Provider';const SANDBOX_NOT_FOUND_STATUS = 404;const SANDBOX_STATE_STARTED = 'started';@@ -57,7 +57,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {private readonly previewUrlExpirySeconds: number;private readonly logger: Logger;private readonly daytona: Daytona;- private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; timeoutMs: number }>();+ private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();// De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip.private static readonly inFlightRecoveries = new Map<string, Promise<boolean>>();@@ -72,7 +72,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {this.daytona = getDaytonaClient(this.apiKey);}- private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; timeoutMs: number }> {+ private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> {if (sandboxId) {const cached = DaytonaSandboxProvider.cachedSandboxes.get(sandboxId);if (cached) return cached;@@ -88,7 +88,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {autoDeleteInterval: this.settings.autoDeleteIntervalInMinutes,});- const entry = { sandbox, timeoutMs: this.settings.timeoutMs };+ const entry = { sandbox, defaultTimeoutMs: this.settings.timeoutMs };DaytonaSandboxProvider.cachedSandboxes.set(sandbox.name, entry);return entry;}@@ -172,21 +172,16 @@ export class DaytonaSandboxProvider implements SandboxProvider {});}- async exec(params: {- sandboxId: string;- command: string;- cwd?: string | undefined;- env?: Record<string, string> | undefined;- }): Promise<ExecResult> {+ async exec(params: SandboxExecParams): Promise<ExecResult> {return context.with(suppressTracing(context.active()), async (): Promise<ExecResult> => {try {return await this.executeWithSandboxRecovery(params.sandboxId, async () => {- const { sandbox, timeoutMs } = await this.getOrCreateSandbox(params.sandboxId);+ const { sandbox, defaultTimeoutMs } = await this.getOrCreateSandbox(params.sandboxId);const response = await sandbox.process.executeCommand(params.command,params.cwd,params.env ?? {},- timeoutMs / 1000,+ params.timeoutSeconds ?? defaultTimeoutMs / 1000,);return {success: true,diff --git a/packages/harness/src/core/sandbox/provider/Provider.ts b/packages/harness/src/core/sandbox/provider/Provider.tsindex 7946a0167..dee346aa4 100644--- a/packages/harness/src/core/sandbox/provider/Provider.ts+++ b/packages/harness/src/core/sandbox/provider/Provider.ts@@ -12,6 +12,15 @@ export interface ExecErrorResult {export type ExecResult = ExecSuccessResult | ExecErrorResult;+/**+ * Wraps a value in single quotes for safe use in a shell command. Inner single quotes are+ * escaped via the standard shell idiom: ' -> '\''. Use for any user-controlled value that is+ * interpolated into an exec command string.+ */+export function shellEscape(value: string): string {+ return `'${value.replace(/'/g, "'\\''")}'`;+}+/** Throws if the exec result indicates failure (infra error or non-zero exit code). */export function ensureExecSuccess(result: ExecResult): void {if (!result.success) {@@ -27,14 +36,22 @@ export interface SandboxFileInfo {isDir: boolean;}+/** Parameters for a single sandbox command. */+export interface SandboxExecParams {+ sandboxId: string;+ command: string;+ cwd?: string | undefined;+ env?: Record<string, string> | undefined;+ /** Overrides the provider's default exec timeout (e.g. for long skill downloads). */+ timeoutSeconds?: number | undefined;+}++// An init command a producer (e.g. a skill mounter) hands to the Sandbox, which supplies `sandboxId`.+export type SandboxInit = Omit<SandboxExecParams, 'sandboxId'>;+export interface SandboxProvider {createSandbox(): Promise<{ sandboxId: string }>;- exec(params: {- sandboxId: string;- command: string;- cwd?: string | undefined;- env?: Record<string, string> | undefined;- }): Promise<ExecResult>;+ exec(params: SandboxExecParams): Promise<ExecResult>;/** Provider-specific instructions appended to the agent system prompt. */getAdditionalInstructions(): string | undefined;/** Directory inside the sandbox where large tool responses are dumped. */diff --git a/packages/harness/src/core/sandbox/provider/TFYSandboxProvider.ts b/packages/harness/src/core/sandbox/provider/TFYSandboxProvider.tsindex c7ade4018..ed3d0a057 100644--- a/packages/harness/src/core/sandbox/provider/TFYSandboxProvider.ts+++ b/packages/harness/src/core/sandbox/provider/TFYSandboxProvider.ts@@ -5,7 +5,13 @@ import dedent from 'dedent';import type { Logger } from 'winston';import { extractErrorLogFields } from '../../util/errorLogFields';import { SandboxFileNotFoundError, SandboxFileTooLargeError, SandboxPathIsDirectoryError } from '../SandboxErrors';-import { ensureExecSuccess, type ExecResult, type SandboxFileInfo, type SandboxProvider } from './Provider';+import {+ ensureExecSuccess,+ type ExecResult,+ type SandboxExecParams,+ type SandboxFileInfo,+ type SandboxProvider,+} from './Provider';const DEFAULT_TIMEOUT_SECONDS = 60;// Buffer for network latency + response processing on top of the server-side timeout.@@ -22,7 +28,7 @@ export interface TFYSandboxProviderOptions {natsBridgeUrl: string;tenantName: string;fileMaxBytes: number;- execTimeoutMs?: number | undefined;+ defaultExecTimeoutMs?: number | undefined;logger: Logger;}@@ -36,7 +42,7 @@ export class TFYSandboxProvider implements SandboxProvider {private readonly natsBridgeUrl: string;private readonly tenantName: string;private readonly fileMaxBytes: number;- private readonly execTimeoutSeconds: number;+ private readonly defaultExecTimeoutSeconds: number;private readonly logger: Logger;constructor(options: TFYSandboxProviderOptions) {@@ -44,7 +50,7 @@ export class TFYSandboxProvider implements SandboxProvider {this.natsBridgeUrl = options.natsBridgeUrl;this.tenantName = options.tenantName;this.fileMaxBytes = options.fileMaxBytes;- this.execTimeoutSeconds = Math.ceil((options.execTimeoutMs ?? DEFAULT_TIMEOUT_SECONDS * 1000) / 1000);+ this.defaultExecTimeoutSeconds = Math.ceil((options.defaultExecTimeoutMs ?? DEFAULT_TIMEOUT_SECONDS * 1000) / 1000);this.logger = options.logger.child({ module: 'TFYSandboxProvider' });}@@ -54,23 +60,21 @@ export class TFYSandboxProvider implements SandboxProvider {return Promise.resolve({ sandboxId });}- async exec(params: {- sandboxId: string;- command: string;- cwd?: string | undefined;- env?: Record<string, string> | undefined;- }): Promise<ExecResult> {+ async exec(params: SandboxExecParams): Promise<ExecResult> {return context.with(suppressTracing(context.active()), async (): Promise<ExecResult> => {+ // Honor the per-call timeout (e.g. the longer skill-download timeout) for both the server-side+ // request timeout and the client abort timer; fall back to the provider default otherwise.+ const timeoutSeconds = params.timeoutSeconds ?? this.defaultExecTimeoutSeconds;const body = {sandbox_id: params.sandboxId,command: params.command,cwd: params.cwd,env: params.env,- timeout: this.execTimeoutSeconds,+ timeout: timeoutSeconds,};const controller = new AbortController();- const clientTimeoutMs = (this.execTimeoutSeconds + CLIENT_TIMEOUT_BUFFER_SECONDS) * 1000;+ const clientTimeoutMs = (timeoutSeconds + CLIENT_TIMEOUT_BUFFER_SECONDS) * 1000;const timer = setTimeout(() => {controller.abort();}, clientTimeoutMs);@@ -93,11 +97,8 @@ export class TFYSandboxProvider implements SandboxProvider {return result;} catch (e: unknown) {if (e instanceof Error && e.name === 'AbortError') {- this.logger.error(- `Sandbox exec timed out after ${String(this.execTimeoutSeconds)}s`,- extractErrorLogFields(e),- );- return { success: false, error: `Sandbox exec timed out after ${String(this.execTimeoutSeconds)}s` };+ this.logger.error(`Sandbox exec timed out after ${String(timeoutSeconds)}s`, extractErrorLogFields(e));+ return { success: false, error: `Sandbox exec timed out after ${String(timeoutSeconds)}s` };}this.logger.error('Sandbox exec failed', extractErrorLogFields(e));const message = e instanceof Error ? e.message : 'Unknown error';diff --git a/packages/harness/src/core/sandbox/scripts/git_downloader.py b/packages/harness/src/core/sandbox/scripts/git_downloader.pynew file mode 100644index 000000000..259f1719b--- /dev/null+++ b/packages/harness/src/core/sandbox/scripts/git_downloader.py@@ -0,0 +1,474 @@+#!/usr/bin/env python3++# /// script+# dependencies = ["pydantic==2.12.5"]+# ///++"""Materialize git skill directories into TFY_SKILLS_DIR (default /opt/tfy/skills). Requires `git`.++Each git skill (AGENT_GIT_SKILLS) is materialized with a SHA-pinned, blob-filtered sparse `git`+clone that fetches only the requested subdir at the requested commit, keyed by name. A single on-disk+state file records the installed commit + subdir per name so a skill whose ref hasn't moved (same+commit and subdir) is skipped instead of re-cloning every run, and skills no longer desired are+pruned from disk. A sparse clone is used instead of a full repo tarball because its cost is+~constant in the subdir size rather than the whole-repo size, which is dramatically faster for a+small skill living in a large monorepo.++Optional env:+ - AGENT_GIT_SKILLS (base64-encoded JSON list of {name, clone_url, subdir, commit_sha}; empty+ clears git downloads)+ - TFY_SKILLS_DIR: override the skills directory (default /opt/tfy/skills).+"""++from __future__ import annotations++import base64+import json+import os+import re+import shutil+import subprocess+import sys+import tempfile+from pathlib import Path++from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator++DEFAULT_SKILLS_DIR = "/opt/tfy/skills"+STATE_FILE_NAME = ".tfy-skill-downloader-state.json"+# Per-git-invocation wall-clock cap so a hung/slow fetch can't stall sandbox init indefinitely.+GIT_CLONE_TIMEOUT_SECONDS = 120+# Cap the installed skill size so a huge repo-root skill can't fill the persistent skills dir. The+# sparse clone already bounds a subdir skill to its subdir; this also guards the whole-repo (root+# subdir) case where checkout hydrates every file.+GIT_SKILL_MAX_BYTES = 200 * 1024 * 1024 # 200MB installed+# A resolved git object id: sha1 (40 hex) or sha256 (64 hex). Validated here because commit_sha is+# passed as a `git fetch` argument, so it must never be interpretable as an option or injected arg.+COMMIT_SHA_RE = re.compile(r"^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$")+SKILLS_ROOT = Path(os.environ.get("TFY_SKILLS_DIR", DEFAULT_SKILLS_DIR))+STATE_PATH = SKILLS_ROOT / STATE_FILE_NAME+++class GitSkillError(Exception):+ """Recoverable per-repo/per-skill git failure. Raised (not sys.exit) so a single bad repo or+ subdirectory doesn't abort the remaining repository groups (best-effort failure isolation)."""+++def skill_dir(name: str) -> Path | None:+ """Resolve a skill's on-disk directory. This is the ONLY place a skill name becomes a path, so+ the safety check here is what guarantees that no name — freshly requested or read back from the+ (possibly corrupt) state file — can escape SKILLS_ROOT during delete/copy.++ Returns None for an unsafe name (empty, absolute, containing a path separator, "." / "..", or the+ reserved state-file name) so each caller can skip it per its own failure contract instead of+ touching an out-of-tree path. The state-file name is reserved because a skill whose name equals it+ would resolve to STATE_PATH and let a skill install clobber the reconciliation state.+ """+ if not name or name in (".", "..") or "/" in name or "\\" in name or name == STATE_FILE_NAME:+ return None+ return SKILLS_ROOT / name+++class GitSkill(BaseModel):+ """A git skill to materialize: its sandbox dir name, the repo clone URL, the subdir within the+ repo, and the resolved commit SHA. The clone URL and SHA are passed to `git`, so both are format-+ validated here (defense in depth — the gateway already validates and builds them) to keep an+ unexpected value from being interpreted as a git option or reaching a non-public host."""++ model_config = ConfigDict(extra="ignore")++ name: str+ clone_url: str+ subdir: str = ""+ commit_sha: str++ @field_validator("clone_url")+ @classmethod+ def _validate_clone_url(cls, v: str) -> str:+ # Only the two public hosts the gateway supports. Anchoring on the "https://<host>/" prefix+ # blocks non-https schemes (file://, ext::, ssh) and any other host, so this URL can't be+ # turned into an SSRF/local-file/command vector once it reaches `git`.+ if not (v.startswith("https://github.com/") or v.startswith("https://gitlab.com/")):+ raise ValueError(f"git skill clone_url must be a github.com/gitlab.com https URL: {v!r}")+ return v++ @field_validator("commit_sha")+ @classmethod+ def _validate_commit_sha(cls, v: str) -> str:+ if not COMMIT_SHA_RE.match(v):+ raise ValueError(f"git skill commit_sha must be a 40- or 64-char hex object id: {v!r}")+ return v++ @field_validator("subdir")+ @classmethod+ def _validate_subdir(cls, v: str) -> str:+ # Relative subpath within the repo. Reject absolute paths and ".." segments so it can't+ # escape the repo root. Leading/trailing slashes are stripped (repo root == ""). A leading+ # "-" is safe because subdirs are fed to `git sparse-checkout set` via stdin, never as args.+ if v.startswith("/"):+ raise ValueError(f"git skill subdir must be relative: {v!r}")+ stripped = v.strip("/")+ if stripped and ".." in stripped.split("/"):+ raise ValueError(f"git skill subdir must not contain '..': {v!r}")+ return stripped+++class DownloadedGitSkill(BaseModel):+ """A git skill recorded on disk: its dir name, the commit it was installed at, and the repo+ subdir it was installed from. The commit lets a later run skip re-downloading when the requested+ commit is unchanged; the subdir is part of the identity so that changing the subdir at the same+ commit still triggers a re-download instead of serving the previously-installed layout."""++ model_config = ConfigDict(extra="ignore")++ name: str+ # Empty when migrated from a pre-commit-tracking state file — treated as "unknown", forcing a+ # one-time re-download while still keeping the name tracked for pruning.+ commit_sha: str = ""+ # Repo subdir the skill was installed from (repo root == ""). Empty on entries written before+ # subdir tracking; a mismatch against the requested subdir forces a one-time re-download.+ subdir: str = ""+++class SkillDownloaderState(BaseModel):+ """Persisted next to skills; tracks what was downloaded so undesired skills can be pruned."""++ model_config = ConfigDict(extra="ignore")++ downloaded_git_skills: list[DownloadedGitSkill] = Field(+ default_factory=list,+ description="Git skills successfully downloaded (name + installed commit SHA).",+ )+++def load_state() -> SkillDownloaderState:+ if not STATE_PATH.is_file():+ return SkillDownloaderState()+ try:+ raw = json.loads(STATE_PATH.read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return SkillDownloaderState()+ if not isinstance(raw, dict):+ return SkillDownloaderState()+ # Migrate a pre-commit-tracking state file: its `downloaded_git_names` (list of names, no commit)+ # becomes entries with an empty commit_sha. That keeps the names tracked so undesired skills are+ # still pruned, and forces a one-time re-download (empty != any requested commit). Skipped when+ # the new field is already present so we never clobber real commits.+ if "downloaded_git_skills" not in raw and isinstance(raw.get("downloaded_git_names"), list):+ raw["downloaded_git_skills"] = [+ {"name": n, "commit_sha": ""} for n in raw["downloaded_git_names"] if isinstance(n, str)+ ]+ try:+ return SkillDownloaderState.model_validate(raw)+ except ValidationError:+ return SkillDownloaderState()+++def save_state(state: SkillDownloaderState) -> None:+ STATE_PATH.parent.mkdir(parents=True, exist_ok=True)+ tmp = STATE_PATH.with_name(STATE_PATH.name + ".tmp")+ tmp.write_text(state.model_dump_json(indent=2), encoding="utf-8")+ os.replace(tmp, STATE_PATH)+++def _rmtree(path: Path, ignore_errors: bool = True) -> None:+ try:+ shutil.rmtree(path, ignore_errors=ignore_errors)+ except OSError:+ pass+++def _delete_skill_dir_by_name(name: str, label: str) -> None:+ """Fail-closed prune of a no-longer-desired skill's directory: abort the whole run if removal+ fails so we never silently leave a stale skill on disk. skill_dir() returns None for a+ corrupt/tampered name, so rmtree never runs on an out-of-tree path."""+ dir_ = skill_dir(name)+ if dir_ is None or not dir_.is_dir():+ return+ try:+ shutil.rmtree(dir_)+ except OSError as e:+ sys.exit(f"Could not remove skill directory {dir_} ({label}): {e}")+++def git_skills_from_env(raw: str | None) -> list[GitSkill]:+ """Empty or missing env means "no desired git skills" (prune all git downloads)."""+ if raw is None or not str(raw).strip():+ return []+ try:+ data = json.loads(base64.b64decode(raw))+ except (ValueError, json.JSONDecodeError) as e:+ sys.exit(f"AGENT_GIT_SKILLS is not valid base64-encoded JSON: {e}")+ if not isinstance(data, list):+ sys.exit("AGENT_GIT_SKILLS must be a JSON list.")+ try:+ return [GitSkill.model_validate(item) for item in data]+ except ValidationError as e:+ sys.exit(f"Invalid AGENT_GIT_SKILLS entry: {e}")+++def reconcile_git_skills(git_skills: list[GitSkill], state: SkillDownloaderState) -> list[str]:+ """Delete dirs for git skills no longer desired; prune them from state. Returns removed names."""+ desired_names = {g.name for g in git_skills}+ removed: list[str] = []+ for entry in state.downloaded_git_skills:+ if entry.name in desired_names:+ continue+ # Fail-closed prune: a failed delete aborts the run rather than leaving a stale skill on disk.+ # The name is dropped from tracking either way.+ _delete_skill_dir_by_name(entry.name, entry.name)+ removed.append(entry.name)+ state.downloaded_git_skills = [+ e for e in state.downloaded_git_skills if e.name in desired_names+ ]+ return removed+++def _run_git(+ args: list[str], cwd: Path, input_text: str | None = None+) -> subprocess.CompletedProcess[str]:+ """Run a git subcommand non-interactively and fail-closed on error.++ Args are passed as a list (never a shell string) and git runs with a scrubbed environment:+ ambient global/system config and credential helpers are ignored (so nothing can rewrite the URL+ via insteadOf or inject creds) and prompting is disabled (so a private/typo repo fails fast+ instead of hanging on a username prompt)."""+ env = {+ **os.environ,+ "GIT_TERMINAL_PROMPT": "0", # never prompt for credentials — fail fast on a non-public repo+ "GIT_CONFIG_NOSYSTEM": "1", # ignore /etc/gitconfig+ "GIT_CONFIG_GLOBAL": "/dev/null", # ignore ~/.gitconfig (insteadOf rewrites, cred helpers)+ "GCM_INTERACTIVE": "never",+ }+ try:+ proc = subprocess.run(+ ["git", *args],+ cwd=os.fspath(cwd),+ env=env,+ input=input_text,+ capture_output=True,+ text=True,+ timeout=GIT_CLONE_TIMEOUT_SECONDS,+ )+ except subprocess.TimeoutExpired:+ raise GitSkillError(f"git {args[0]} timed out after {GIT_CLONE_TIMEOUT_SECONDS}s")+ except OSError as e:+ raise GitSkillError(f"could not run git (is it installed in the sandbox?): {e}")+ if proc.returncode != 0:+ detail = (proc.stderr or proc.stdout or "").strip()[-500:]+ raise GitSkillError(f"git {args[0]} failed (exit {proc.returncode}): {detail}")+ return proc+++def _git_fetch_repo(+ clone_url: str, commit_sha: str, subdirs: list[str], parent_dir: Path+) -> Path:+ """SHA-pinned, shallow, blob-filtered sparse checkout of `subdirs` into a fresh repo under+ `parent_dir`, returning the repo working directory.++ When every requested subdir is the repo root ("") we can't sparse-restrict, so we do a full+ shallow checkout (all blobs). Otherwise we cone-sparse to just the requested subdirs and use+ --filter=blob:none so only those subdirs' blobs are fetched — the whole point of the approach.+ Subdirs are fed to `git sparse-checkout set` via stdin so a subdir value can never be read as an+ option/argument. Fetching by object id relies on the server allowing it, which github.com and+ gitlab.com both do."""+ repo = parent_dir / "repo"+ _run_git(["init", "-q", os.fspath(repo)], cwd=parent_dir)+ _run_git(["remote", "add", "origin", clone_url], cwd=repo)++ cone_dirs = [s for s in dict.fromkeys(subdirs) if s] # de-duped, root ("") dropped+ want_full_tree = any(s == "" for s in subdirs)+ if not want_full_tree and cone_dirs:+ _run_git(["sparse-checkout", "init", "--cone"], cwd=repo)+ _run_git(+ ["sparse-checkout", "set", "--stdin"],+ cwd=repo,+ input_text="".join(f"{d}\n" for d in cone_dirs),+ )++ fetch_args = ["-c", "protocol.file.allow=never", "fetch", "-q", "--depth", "1"]+ if not want_full_tree:+ fetch_args.append("--filter=blob:none")+ fetch_args += ["origin", commit_sha]+ _run_git(fetch_args, cwd=repo)++ _run_git(+ ["-c", "advice.detachedHead=false", "checkout", "-q", "--detach", "FETCH_HEAD"], cwd=repo+ )+ return repo+++def _installed_size_bytes(path: Path) -> int:+ """Total size of regular files under `path`, excluding the repo's .git dir and without following+ symlinks (they're copied as links, not their targets), used to enforce GIT_SKILL_MAX_BYTES."""+ total = 0+ for root, dirs, files in os.walk(path, followlinks=False):+ if ".git" in dirs:+ dirs.remove(".git")+ for name in files:+ try:+ total += (Path(root) / name).lstat().st_size+ except OSError:+ continue+ return total+++def _install_git_skill(repo_root: Path, skill: GitSkill) -> None:+ dest = skill_dir(skill.name)+ if dest is None:+ raise GitSkillError(f"Git skill: unsafe directory name {skill.name!r}")+ repo_root_resolved = repo_root.resolve()+ src = (repo_root / skill.subdir).resolve() if skill.subdir else repo_root_resolved+ # Defense in depth: even though subdir is validated, ensure the resolved source stays within the+ # cloned repo (guards against a symlinked directory component pointing outside the checkout).+ if src != repo_root_resolved and not src.is_relative_to(repo_root_resolved):+ raise GitSkillError(+ f"Git skill {skill.name}: subdirectory '{skill.subdir}' escapes the repository"+ )+ if not src.is_dir():+ raise GitSkillError(+ f"Git skill {skill.name}: subdirectory '{skill.subdir}' not found in repository"+ )+ if _installed_size_bytes(src) > GIT_SKILL_MAX_BYTES:+ raise GitSkillError(+ f"Git skill {skill.name}: content exceeds {GIT_SKILL_MAX_BYTES} bytes"+ )+ # Stage the fresh copy into a temp sibling dir, then swap it in with an atomic rename. `dest` is+ # only touched once the copy has fully succeeded, so a failed/partial copytree can never leave+ # the skill missing (which would contradict the path the gateway advertises in the prompt) — the+ # previously installed copy stays intact on failure. `.git` is excluded (only present for a repo-+ # root skill) and symlinks are copied as links, never followed, so a link inside the repo can't+ # pull external file contents into the skill dir at copy time.+ staging = dest.with_name(f".{dest.name}.new-{os.getpid()}")+ _rmtree(staging) # clear any leftover staging dir from an earlier aborted run+ try:+ shutil.copytree(src, staging, symlinks=True, ignore=shutil.ignore_patterns(".git"))+ except (OSError, shutil.Error) as e:+ # shutil.Error (a multi-file copy error, NOT an OSError subclass) must be caught here too so a+ # bad copy is reported as a per-skill GitSkillError (WARNING + counted failure) instead of an+ # uncaught traceback that aborts the whole run and skips save_state for the group.+ _rmtree(staging)+ raise GitSkillError(f"Failed to stage git skill {skill.name}: {e}")+ try:+ if dest.exists():+ shutil.rmtree(dest)+ # Same filesystem (both under SKILLS_ROOT), so this is an atomic rename; the window where the+ # skill dir is absent shrinks from a full copy to a single near-instant rename.+ os.replace(staging, dest)+ except OSError as e:+ _rmtree(staging)+ raise GitSkillError(f"Failed to install git skill {skill.name}: {e}")+++def _mark_git_downloaded(state: SkillDownloaderState, skill: GitSkill) -> None:+ """Upsert a git skill's installed commit + subdir so the next run can skip it when both are unchanged."""+ for entry in state.downloaded_git_skills:+ if entry.name == skill.name:+ entry.commit_sha = skill.commit_sha+ entry.subdir = skill.subdir+ return+ state.downloaded_git_skills.append(+ DownloadedGitSkill(name=skill.name, commit_sha=skill.commit_sha, subdir=skill.subdir)+ )+++def _git_skill_already_installed(state: SkillDownloaderState, skill: GitSkill) -> bool:+ """True only when the skill is recorded at the requested commit AND subdir AND its directory is+ still present. Re-download otherwise (commit moved, subdir changed, first install, or the dir was+ removed out-of-band) so the prompt never points at a missing/stale skill dir."""+ dir_ = skill_dir(skill.name)+ if dir_ is None or not dir_.is_dir():+ return False+ return any(+ e.name == skill.name and e.commit_sha == skill.commit_sha and e.subdir == skill.subdir+ for e in state.downloaded_git_skills+ )+++def download_git_skills(+ git_skills: list[GitSkill], state: SkillDownloaderState+) -> int:+ """Ensure desired git skills are installed at their requested commit, skipping any already present+ at that commit so an unchanged ref no longer re-clones every run.++ Failures are isolated per repo group and per skill so one bad repo/subdir doesn't prevent the+ rest from being attempted; each failure is logged to stderr. Returns the number of skills that are+ satisfied this run — freshly installed OR already present at the requested commit (never raises).+ The caller (run_git_download) compares this to the number requested and exits non-zero if any are+ missing, so the overall behaviour is fail-closed."""+ SKILLS_ROOT.mkdir(parents=True, exist_ok=True)++ # Skip skills already installed at the requested commit; only the rest need a clone.+ pending: list[GitSkill] = []+ satisfied = 0+ for skill in git_skills:+ if _git_skill_already_installed(state, skill):+ satisfied += 1+ else:+ pending.append(skill)++ # Group pending by (repo, commit) so a repo shared by several skills is cloned once and its+ # needed subdirs fetched together in a single sparse checkout.+ skills_by_repo: dict[tuple[str, str], list[GitSkill]] = {}+ for skill in pending:+ skills_by_repo.setdefault((skill.clone_url, skill.commit_sha), []).append(skill)++ for (clone_url, commit_sha), skills in skills_by_repo.items():+ try:+ with tempfile.TemporaryDirectory(+ prefix=".git-skill-dl-",+ dir=os.fspath(SKILLS_ROOT),+ ignore_cleanup_errors=True,+ ) as tmp_str:+ repo_root = _git_fetch_repo(+ clone_url, commit_sha, [s.subdir for s in skills], Path(tmp_str)+ )+ # Install each skill independently so a bad subdir only skips that one skill.+ for skill in skills:+ try:+ _install_git_skill(repo_root, skill)+ _mark_git_downloaded(state, skill)+ satisfied += 1+ except GitSkillError as e:+ print(f"WARNING: {e}", file=sys.stderr)+ except GitSkillError as e:+ # Collect (don't abort-on-first) so EVERY failing skill is reported before the run exits;+ # run_git_download turns a non-zero failed count into a non-zero exit (fail-closed), which+ # fails the whole agent request. We still don't purge a previously installed copy here, so+ # a reused sandbox keeps the old files on disk even though this run is about to fail.+ names = ", ".join(s.name for s in skills)+ print(f"WARNING: {e} (skills: {names})", file=sys.stderr)+ save_state(state)+ return satisfied+++def run_git_download() -> None:+ git_skills = git_skills_from_env(os.environ.get("AGENT_GIT_SKILLS"))+ state = load_state()+ removed = reconcile_git_skills(git_skills, state)+ save_state(state) # persist pruning even when there is nothing to download++ if not git_skills:+ if removed:+ print(f"Removed {len(removed)} git skill(s) from disk. (AGENT_GIT_SKILLS empty.)")+ return++ satisfied = download_git_skills(git_skills, state)+ failed = len(git_skills) - satisfied+ suffix = f" Removed {len(removed)} stale git skill(s)." if removed else ""+ if failed:+ # Fail-closed: a requested skill that couldn't be installed exits non-zero so the gateway+ # (ensureExecSuccess) fails the whole agent request, matching the fail-closed SKILL.md fetch.+ # Per-skill WARNINGs above list exactly what failed.+ sys.exit(+ f"Failed to install {failed}/{len(git_skills)} git skill(s); see warnings above.{suffix}"+ )+ print(f"Ensured {satisfied} git skill(s) (downloaded or already up to date).{suffix}")+++def main() -> None:+ run_git_download()+++if __name__ == "__main__":+ main()diff --git a/packages/harness/src/core/sandbox/scripts/mcp_client.py b/packages/harness/src/core/sandbox/scripts/mcp_client.pyindex 4289bbac5..b0df23e6f 100644--- a/packages/harness/src/core/sandbox/scripts/mcp_client.py+++ b/packages/harness/src/core/sandbox/scripts/mcp_client.py@@ -3,9 +3,6 @@# /// script# dependencies = ["fastmcp==3.2.4", "pydantic==2.12.5", "nats-py==2.15.0"]# ///-# NOTE: If you change or bump dependencies above, update them in these repos too:-# - https://github.com/truefoundry/tfy-llm-tools-# - https://github.com/truefoundry/tfy-sandbox-serverimport osimport sysdiff --git a/packages/harness/src/core/sandbox/scripts/skill_downloader.py b/packages/harness/src/core/sandbox/scripts/skill_downloader.pydeleted file mode 100644index b405190f3..000000000--- a/packages/harness/src/core/sandbox/scripts/skill_downloader.py+++ /dev/null@@ -1,308 +0,0 @@-#!/usr/bin/env python3--# /// script-# dependencies = ["pydantic==2.12.5", "requests>=2.31,<3"]-# ///--"""Download skill tarballs (presigned URLs) into TFY_SKILLS_DIR (default /opt/tfy/skills).--Required env (only when there are skills to download): TFY_HOST, TFY_API_KEY (caller bearer).-Optional env: TFY_AGENT_NAME, AGENT_SKILL_VERSION_FQNS (comma-separated; empty clears downloads).-"""--from __future__ import annotations--import json-import os-import shutil-import sys-import tarfile-import tempfile-from io import BytesIO-from pathlib import Path--import requests-from pydantic import BaseModel, ConfigDict, Field, ValidationError--DEFAULT_SKILLS_DIR = "/opt/tfy/skills"-STATE_FILE_NAME = ".tfy-skill-downloader-state.json"-DOWNLOAD_TIMEOUT_SECONDS = 30-PRESIGNED_URL_PATH = "/api/svc/v1/llm-gateway/agent-skill-versions/presigned-urls"-SKILLS_ROOT = Path(os.environ.get("TFY_SKILLS_DIR", DEFAULT_SKILLS_DIR))-STATE_PATH = SKILLS_ROOT / STATE_FILE_NAME---class AgentSkill(BaseModel):- model_config = ConfigDict(extra="ignore")-- fqn: str- name: str---class SkillDownloaderState(BaseModel):- """Persisted next to skills: {\"downloaded_fqns\": [{\"fqn\": \"agent_skill:...\"]}."""-- model_config = ConfigDict(extra="ignore")-- downloaded_fqns: list[AgentSkill] = Field(- default_factory=list,- description="Skills successfully downloaded (fqn + name).",- )---class PresignedUrlsRequestItem(BaseModel):- model_config = ConfigDict(extra="forbid")-- fqn: str---class PresignedUrlsRequest(BaseModel):- model_config = ConfigDict(extra="forbid", populate_by_name=True)-- data: list[PresignedUrlsRequestItem]- agent_name: str | None = Field(default=None, alias="agentName")---class AgentSkillPresignedRow(AgentSkill):- model_config = ConfigDict(extra="ignore")-- presigned_url: str---class PresignedUrlsResponse(BaseModel):- model_config = ConfigDict(extra="ignore")-- data: list[AgentSkillPresignedRow]---class ReconcilePartitionResult(BaseModel):- """Pending downloads, skipped (already satisfied), and FQNs pruned from state."""-- model_config = ConfigDict(extra="forbid")-- pending_fqns: list[str] = Field(- default_factory=list,- description="FQNs to download this run.",- )- skipped_fqns: list[str] = Field(- default_factory=list,- description="FQNs already satisfied (state or on-disk); skipped.",- )- removed_fqns: list[str] = Field(- default_factory=list,- description="FQNs dropped from desired; removed from state and disk.",- )---def skills_from_env(raw: str | None) -> list[str]:- """Empty or missing env means “no desired skills” (prune all downloaded)."""- if raw is None or not str(raw).strip():- return []- return [s.strip() for s in raw.split(",") if s.strip()]---def load_state() -> SkillDownloaderState:- if not STATE_PATH.is_file():- return SkillDownloaderState()- try:- return SkillDownloaderState.model_validate_json(STATE_PATH.read_text(encoding="utf-8"))- except (OSError, ValidationError) as e:- return SkillDownloaderState()---def save_state(state: SkillDownloaderState) -> None:- STATE_PATH.parent.mkdir(parents=True, exist_ok=True)- tmp = STATE_PATH.with_name(STATE_PATH.name + ".tmp")- tmp.write_text(state.model_dump_json(indent=2), encoding="utf-8")- os.replace(tmp, STATE_PATH)---def _mark_downloaded(state: SkillDownloaderState, skill: AgentSkill) -> None:- if any(x.fqn == skill.fqn for x in state.downloaded_fqns):- return- state.downloaded_fqns.append(skill)---def _rmtree(path: Path, ignore_errors: bool = True) -> None:- try:- shutil.rmtree(path, ignore_errors=ignore_errors)- except OSError:- pass---def _delete_skill_dir(skill: AgentSkill) -> None:- dir_ = SKILLS_ROOT / skill.name- if not dir_.is_dir():- return- try:- shutil.rmtree(dir_)- except OSError as e:- sys.exit(f"Could not remove skill directory {dir_} ({skill.fqn}): {e}")---def reconcile_and_partition(- skills_fqns: list[str], # list of FQNs- state: SkillDownloaderState,- have_downloaded_skills: bool,-) -> ReconcilePartitionResult:- """Drop skills no longer desired (state + dirs), then pending vs skipped; persist state."""- requested_fqns = set(skills_fqns)- pending_fqns: list[str] = []- skipped_fqns: list[str] = []- removed = [x for x in state.downloaded_fqns if x.fqn not in requested_fqns]- removed_fqns = [x.fqn for x in removed]- if removed:- for x in removed:- _delete_skill_dir(x)- state.downloaded_fqns = [x for x in state.downloaded_fqns if x.fqn in requested_fqns]-- if not have_downloaded_skills:- pending_fqns = list(skills_fqns)- else:- already_downloaded = {x.fqn for x in state.downloaded_fqns}- for skill_fqn in requested_fqns:- if skill_fqn in already_downloaded:- skipped_fqns.append(skill_fqn)- else:- pending_fqns.append(skill_fqn)-- save_state(state)- return ReconcilePartitionResult(- pending_fqns=pending_fqns,- skipped_fqns=skipped_fqns,- removed_fqns=removed_fqns,- )---def request_presigned_rows(- control_plane_url: str,- token: str,- agent_name: str | None,- fqns: list[str],-) -> list[AgentSkillPresignedRow]:- url = f"{control_plane_url.rstrip('/')}{PRESIGNED_URL_PATH}"- req = PresignedUrlsRequest(- data=[PresignedUrlsRequestItem(fqn=fqn) for fqn in fqns],- agent_name=agent_name,- )- try:- resp = requests.post(- url,- json=req.model_dump(by_alias=True, exclude_none=True),- headers={"Authorization": f"Bearer {token}"},- timeout=120,- )- except requests.RequestException as e:- sys.exit(f"POST {url} failed: {e}")- if resp.status_code >= 400:- try:- err = resp.json()- detail = err.get("message", resp.text)- if code := err.get("error_code"):- detail = f"{code}: {detail}"- except (json.JSONDecodeError, ValueError):- detail = resp.text- sys.exit(f"POST {url} failed HTTP {resp.status_code}\n{detail}")- try:- payload = resp.json()- except json.JSONDecodeError as e:- sys.exit(f"POST {url}: invalid JSON in response: {e}")- try:- rows = PresignedUrlsResponse.model_validate(payload).data- except ValidationError as e:- sys.exit(f"Invalid API response: {e}")-- returned_fqns = {r.fqn for r in rows}- if miss := [f for f in fqns if f not in returned_fqns]:- sys.exit(f"API did not return these FQNs: {miss}")- return rows---def download_skill(skill_name: str, presigned_url: str, dest: Path) -> None:- try:- r = requests.get(presigned_url, timeout=DOWNLOAD_TIMEOUT_SECONDS)- r.raise_for_status()- raw = r.content- except requests.HTTPError as e:- code = e.response.status_code if e.response is not None else "?"- sys.exit(f"Download HTTP {code} for skill {skill_name}")- except requests.RequestException as e:- sys.exit(f"Download failed for skill {skill_name}: {e}")-- dest.parent.mkdir(parents=True, exist_ok=True)- with tempfile.TemporaryDirectory(- prefix=".skill-dl-",- dir=os.fspath(dest.parent),- ignore_cleanup_errors=True,- ) as tmp_str:- tmp_dir = Path(tmp_str)- try:- tf = tarfile.open(fileobj=BytesIO(raw), mode="r:*")- except (tarfile.TarError, OSError, EOFError):- sys.exit(f"Skill {skill_name}: not a tar archive")- try:- with tf:- tf.extractall(tmp_dir, filter="data")- except tarfile.TarError as e:- sys.exit(f"Skill {skill_name}: tar extraction refused or failed ({e})")- except OSError as e:- sys.exit(f"Skill {skill_name}: failed to read archive: {e}")-- try:- if dest.exists():- shutil.rmtree(dest)- shutil.move(str(tmp_dir), str(dest))- except OSError as e:- _rmtree(dest)- sys.exit(f"Failed to write downloaded skill {skill_name}: {e}")---def run_download() -> None:- skills_fqns = skills_from_env(os.environ.get("AGENT_SKILL_VERSION_FQNS"))- state = load_state()- have_downloaded_skills = bool(state.downloaded_fqns)-- part = reconcile_and_partition(- skills_fqns, state, have_downloaded_skills=have_downloaded_skills- )- if not part.pending_fqns:- if part.removed_fqns:- n = len(part.removed_fqns)- print(- f"Removed {n} skill(s) from state and disk."- + ("" if skills_fqns else " (AGENT_SKILL_VERSION_FQNS empty.)")- )- elif not skills_fqns:- print("AGENT_SKILL_VERSION_FQNS empty; nothing downloaded.")- else:- print("Nothing to download; all listed skills already downloaded.")- return-- # Credentials are only needed when there is actually something to download,- # so runtimes with no skills configured (e.g. the public server) can init- # the sandbox without a control plane.- control_plane_url = os.environ.get("TFY_HOST")- if not control_plane_url:- sys.exit("Set TFY_HOST to the control plane base URL.")- token = os.environ.get("TFY_API_KEY")- if not token:- sys.exit("Set TFY_API_KEY.")- agent_name = os.environ.get("TFY_AGENT_NAME")-- rows = request_presigned_rows(- control_plane_url, token, agent_name, part.pending_fqns- )- for row in rows:- download_skill(row.name, row.presigned_url, SKILLS_ROOT / row.name)- _mark_downloaded(state, AgentSkill(fqn=row.fqn, name=row.name))- save_state(state)-- print(f"Downloaded {len(rows)} skill(s).")---def main() -> None:- run_download()---if __name__ == "__main__":- main()diff --git a/packages/harness/src/core/sandbox/skills/ISkillMounter.ts b/packages/harness/src/core/sandbox/skills/ISkillMounter.tsnew file mode 100644index 000000000..afacd62c8--- /dev/null+++ b/packages/harness/src/core/sandbox/skills/ISkillMounter.ts@@ -0,0 +1,11 @@+import type { InstructionBuilder } from '../../InstructionBuilder';+import type { SandboxInit } from '../provider/Provider';++// A set of skills that describes itself in the prompt and installs itself into a sandbox. The sandbox+// holds one mounter and never inspects individual skills.+export interface ISkillMounter {+ // Renders the <skills> section; adds nothing when empty, so InstructionBuilder drops the section.+ instruction(builder: InstructionBuilder): void;+ // The command (+ env, timeout) that installs these skills; Sandbox folds it into its init exec.+ getSandboxInit(): SandboxInit;+}diff --git a/packages/harness/src/core/sandbox/skills/SkillMounter.ts b/packages/harness/src/core/sandbox/skills/SkillMounter.tsnew file mode 100644index 000000000..b45f022bb--- /dev/null+++ b/packages/harness/src/core/sandbox/skills/SkillMounter.ts@@ -0,0 +1,74 @@+import type { InstructionBuilder } from '../../InstructionBuilder';+import type { SandboxInit } from '../provider/Provider';+import { SKILL_DOWNLOAD_TIMEOUT_SECONDS, buildWriteAndRunScriptCommand } from '../Sandbox';+import { sandboxScripts } from '../sandboxScripts.gen';+import { SKILLS_PREAMBLE, getSkillPath, renderSkillPromptBody } from './constants';+import type { ISkillMounter } from './ISkillMounter';++// Absolute path the git skill downloader script is uploaded to before it is run.+const GIT_DOWNLOADER_PATH = '/opt/tfy/git_downloader.py';++// A git-sourced skill materialized by a SHA-pinned sparse clone. Git skills never preload — their+// SKILL.md is read from disk at runtime (only name/description/path are advertised in the prompt).+export interface GitSkill {+ readonly name: string;+ readonly description: string;+ // Canonical https clone URL of the github.com/gitlab.com repo (host-validated by the caller).+ readonly cloneUrl: string;+ // Subdirectory within the repo that holds the skill (empty = repo root).+ readonly subdir: string;+ // Immutable commit the sparse clone is pinned to; an unchanged commit is skipped on later runs.+ readonly commitSha: string;+}++// Serializable descriptor handed to git_downloader.py (base64 JSON); keys match its GitSkill model.+interface GitSkillSpec {+ name: string;+ clone_url: string;+ subdir: string;+ commit_sha: string;+}++export class SkillMounter implements ISkillMounter {+ private readonly skills: readonly GitSkill[];++ constructor(skills: readonly GitSkill[]) {+ this.skills = skills;+ }++ instruction(builder: InstructionBuilder): void {+ if (this.skills.length === 0) return;+ builder.addContent(SKILLS_PREAMBLE);+ for (const skill of this.skills) {+ builder.addSection(+ 'skill',+ renderSkillPromptBody({+ path: getSkillPath(skill.name),+ name: skill.name,+ description: skill.description,+ preloadContent: null,+ }),+ );+ }+ }++ getSandboxInit(): SandboxInit {+ // An empty desired set is also the source-neutral cleanup path for a reused sandbox.+ const specs: GitSkillSpec[] = this.skills.map(skill => ({+ name: skill.name,+ clone_url: skill.cloneUrl,+ subdir: skill.subdir,+ commit_sha: skill.commitSha,+ }));+ const gitSkillsB64 = Buffer.from(JSON.stringify(specs)).toString('base64');+ return {+ command: buildWriteAndRunScriptCommand({+ scriptPath: GIT_DOWNLOADER_PATH,+ // Bundled at build time (sandboxScripts.gen.ts) so the packaged library has no loose files.+ scriptContent: sandboxScripts.gitDownloader,+ }),+ env: { AGENT_GIT_SKILLS: gitSkillsB64 },+ timeoutSeconds: SKILL_DOWNLOAD_TIMEOUT_SECONDS,+ };+ }+}diff --git a/packages/harness/src/core/sandbox/skills/constants.ts b/packages/harness/src/core/sandbox/skills/constants.tsnew file mode 100644index 000000000..8ef328f61--- /dev/null+++ b/packages/harness/src/core/sandbox/skills/constants.ts@@ -0,0 +1,31 @@+// Skill prompt constants and helpers: how skills are described to the agent in the system prompt.++// Directory inside the sandbox where every skill (whatever its origin) is materialized.+export const SKILLS_DIR = '/opt/tfy/skills';++// On-disk directory a skill is materialized into, keyed by its (unique) name.+export function getSkillPath(skillName: string): string {+ return `${SKILLS_DIR}/${skillName}`;+}++export const SKILLS_PREAMBLE = [+ '1. The skill is present in the `path` directory.',+ '2. SKILL.md file within the directory is the primary file, there can be other files as well.',+ '3. Use `description` to judge whether the Agent needs to use a skill.',+ '4. To use a skill:',+ '4.a. If the `SKILL.md content` is inlined, the Agent must not read `path/SKILL.md`.',+ '4.b. Otherwise the Agent must read `path/SKILL.md` first.',+ '5. The Agent must avoid listing the `path` unless there are references in `SKILL.md` and there is reason to do so given the current conversation.',+].join('\n');++// The <skill> block body: inline preloaded SKILL.md when present, else advertise path/name/description.+export function renderSkillPromptBody(params: {+ path: string;+ name: string;+ description: string;+ preloadContent: string | null;+}): string {+ return params.preloadContent+ ? `path: ${params.path}\nSKILL.md content:\n${params.preloadContent}`+ : `path: ${params.path}\nname: ${params.name}\ndescription: ${params.description}`;+}diff --git a/packages/harness/src/core/sandbox/skills/index.ts b/packages/harness/src/core/sandbox/skills/index.tsnew file mode 100644index 000000000..4940b6927--- /dev/null+++ b/packages/harness/src/core/sandbox/skills/index.ts@@ -0,0 +1,4 @@+export { SKILLS_DIR, SKILLS_PREAMBLE, getSkillPath, renderSkillPromptBody } from './constants';+export type { ISkillMounter } from './ISkillMounter';+export { SkillMounter } from './SkillMounter';+export type { GitSkill } from './SkillMounter';diff --git a/packages/harness/tests/core/skillMounter.test.ts b/packages/harness/tests/core/skillMounter.test.tsnew file mode 100644index 000000000..547883031--- /dev/null+++ b/packages/harness/tests/core/skillMounter.test.ts@@ -0,0 +1,51 @@+/**+ * Public git-based SkillMounter: prompt rendering and the declarative sandbox+ * init it hands to Sandbox. (Host-specific mounters, e.g. the gateway's+ * TrueFoundry registry mounter, are tested in their own repo.)+ */+import { InstructionBuilder } from '../../src/core/InstructionBuilder';+import type { ISkillMounter } from '../../src/core/sandbox/skills/ISkillMounter';+import { SkillMounter } from '../../src/core/sandbox/skills/SkillMounter';++const GIT_SKILL = {+ name: 'git-skill',+ description: 'From git',+ cloneUrl: 'https://github.com/acme/skills.git',+ subdir: '',+ commitSha: 'a'.repeat(40),+};++// Renders a mounter's <skills> section the same way Sandbox does (empty section => '').+function renderSkills(mounter: ISkillMounter): string {+ const builder = new InstructionBuilder('skills');+ mounter.instruction(builder);+ return builder.build();+}++describe('SkillMounter (public, git-only)', () => {+ it('embeds the git downloader and passes only git skills', () => {+ const init = new SkillMounter([GIT_SKILL]).getSandboxInit();++ expect(init.command).toContain('/opt/tfy/git_downloader.py');+ expect(init.env?.['AGENT_GIT_SKILLS']).toBeDefined();+ expect(init.env?.['AGENT_SKILL_VERSION_FQNS']).toBeUndefined();+ expect(init.env?.['TFY_API_KEY']).toBeUndefined();+ expect(init.timeoutSeconds).toBe(180);+ });++ it('renders a <skills> section with a block per skill', () => {+ const rendered = renderSkills(new SkillMounter([GIT_SKILL]));++ expect(rendered).toContain('<skills>');+ expect(rendered).toContain('git-skill');+ });++ it('uses an empty desired set for cleanup on a reused sandbox and renders nothing', () => {+ const mounter = new SkillMounter([]);+ const init = mounter.getSandboxInit();++ expect(renderSkills(mounter)).toBe('');+ expect(init.command).toContain('/opt/tfy/git_downloader.py');+ expect(JSON.parse(Buffer.from(init.env?.['AGENT_GIT_SKILLS'] ?? '', 'base64').toString())).toEqual([]);+ });+});diff --git a/packages/server/src/runtime/sandboxFactory.ts b/packages/server/src/runtime/sandboxFactory.tsindex e6a6a186e..0a1e11f59 100644--- a/packages/server/src/runtime/sandboxFactory.ts+++ b/packages/server/src/runtime/sandboxFactory.ts@@ -66,12 +66,8 @@ export function createServerSandboxFactory(deps: { logger: Logger }): TurnSandbologger,});- // TODO(skills): wire the complete skills flow. Spec-declared skills (from- // skills.yaml) should be resolved to MountedSkill[] and passed as- // `mountedSkills`, with `skillInitEnv` carrying the credentials- // skill_downloader.py needs (TFY_HOST, TFY_API_KEY, TFY_AGENT_NAME) to- // download tarballs into the sandbox — mirroring the private gateway's- // sandboxComposition.ts. Until then sandbox sessions run without skills.+ // TODO(skills): wire public-server skill specs to an ISkillMounter (e.g. the git-based+ // SkillMounter). Until then, public-server sandbox sessions run without skills.return ({ spec, existingSandboxId, tracing }) =>Promise.resolve(new Sandbox({