Open source file-system and knowledge based agent template. Build AI agents that stay up to date with your knowledge base
use sandboxes in workflow
Hugo Richard656b4f530620
diff --git a/apps/api/routes/api/sync/[source].post.ts b/apps/api/routes/api/sync/[source].post.tsindex da296d0..8ce35d9 100644--- a/apps/api/routes/api/sync/[source].post.ts+++ b/apps/api/routes/api/sync/[source].post.ts@@ -1,5 +1,5 @@import { start } from 'workflow/api'-import { defineHandler, readValidatedBody, getValidatedRouterParams } from 'nitro/h3'+import { defineHandler, getValidatedRouterParams } from 'nitro/h3'import { useRuntimeConfig } from 'nitro/runtime-config'import { z } from 'zod'import { syncDocumentation } from '~/workflows/sync-docs'@@ -8,27 +8,15 @@ const paramsSchema = z.object({source: z.string().min(1),})-const bodySchema = z- .object({- reset: z.boolean().default(false),- push: z.boolean().default(true),- })- .optional()-/*** POST /api/sync/:source- * Sync a specific source.+ * Sync a specific source using Vercel Sandbox.** Params:* - source: Source ID to sync- *- * Body (optional):- * - reset: boolean - Clear content before sync (default: false)- * - push: boolean - Push to snapshot repo after sync (default: true)*/export default defineHandler(async (event) => {const { source } = await getValidatedRouterParams(event, paramsSchema.parse)- const body = await readValidatedBody(event, data => bodySchema.parse(data))const config = useRuntimeConfig()const syncConfig = {@@ -38,8 +26,6 @@ export default defineHandler(async (event) => {}const options = {- reset: body?.reset ?? false,- push: body?.push ?? true,sourceFilter: source,}@@ -49,6 +35,5 @@ export default defineHandler(async (event) => {status: 'started',message: `Sync workflow started for source "${source}". Use \`pnpm workflow:web\` to monitor.`,source,- options,}})diff --git a/apps/api/routes/api/sync/index.post.ts b/apps/api/routes/api/sync/index.post.tsindex d7883ac..e773f68 100644--- a/apps/api/routes/api/sync/index.post.ts+++ b/apps/api/routes/api/sync/index.post.ts@@ -6,18 +6,16 @@ import { syncDocumentation } from '~/workflows/sync-docs'const bodySchema = z.object({- reset: z.boolean().default(false),- push: z.boolean().default(true),+ sourceFilter: z.string().optional(),}).optional()/*** POST /api/sync- * Sync all sources.+ * Sync all sources using Vercel Sandbox.** Body (optional):- * - reset: boolean - Clear all content before sync (default: false)- * - push: boolean - Push to snapshot repo after sync (default: true)+ * - sourceFilter: string - Only sync a specific source*/export default defineHandler(async (event) => {const body = await readValidatedBody(event, data => bodySchema.parse(data))@@ -30,8 +28,7 @@ export default defineHandler(async (event) => {}const options = {- reset: body?.reset ?? false,- push: body?.push ?? true,+ sourceFilter: body?.sourceFilter,}await start(syncDocumentation, [syncConfig, options])@@ -39,6 +36,5 @@ export default defineHandler(async (event) => {return {status: 'started',message: 'Sync workflow started. Use `pnpm workflow:web` to monitor.',- options,}})diff --git a/apps/api/workflows/create-snapshot/workflow.ts b/apps/api/workflows/create-snapshot/workflow.tsindex 4321dfd..53cfb31 100644--- a/apps/api/workflows/create-snapshot/workflow.ts+++ b/apps/api/workflows/create-snapshot/workflow.ts@@ -87,7 +87,7 @@ async function storeSnapshotMetadataStep(const isProduction = !!(process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN)if (isProduction) {- // Use Upstash Redis in production+ // eslint-disable-next-lineconst { Redis } = await import('@upstash/redis')const redis = new Redis({url: process.env.KV_REST_API_URL!,@@ -95,8 +95,7 @@ async function storeSnapshotMetadataStep(})await redis.set('snapshot:current', metadata)logger.log('snapshot', 'Snapshot metadata stored in Upstash Redis')- }- else {+ } else {// Use filesystem in developmentconst kvDir = join(process.cwd(), '.data', 'kv')const filePath = join(kvDir, 'snapshot:current.json')diff --git a/apps/api/workflows/sync-docs/steps/cleanup.ts b/apps/api/workflows/sync-docs/steps/cleanup.tsdeleted file mode 100644index 911d9d2..0000000--- a/apps/api/workflows/sync-docs/steps/cleanup.ts+++ /dev/null@@ -1,15 +0,0 @@-import { rm } from 'node:fs/promises'-import { getLogger } from '@savoir/logger'--export async function cleanupWorkspace(syncDir: string): Promise<void> {- 'use step'-- const logger = getLogger()-- try {- await rm(syncDir, { recursive: true, force: true })- logger.log('sync', 'Workspace cleaned up')- } catch {- logger.log('sync', 'Failed to cleanup workspace')- }-}diff --git a/apps/api/workflows/sync-docs/steps/index.ts b/apps/api/workflows/sync-docs/steps/index.tsindex 0152c7e..8d61685 100644--- a/apps/api/workflows/sync-docs/steps/index.ts+++ b/apps/api/workflows/sync-docs/steps/index.ts@@ -1,6 +1,4 @@-export { prepareWorkspace } from './prepare-workspace.js'export { getSourcesToSync } from './get-sources.js'-export { syncSingleSource } from './sync-source.js'-export { pushToSnapshotStep } from './push-snapshot.js'-export { cleanupWorkspace } from './cleanup.js'-export { triggerSnapshotStep } from './trigger-snapshot.js'++// Sandbox-based step (single step to avoid serialization issues)+export { syncAllSourcesInSandbox } from './sandbox-sync.js'diff --git a/apps/api/workflows/sync-docs/steps/prepare-workspace.ts b/apps/api/workflows/sync-docs/steps/prepare-workspace.tsdeleted file mode 100644index 09ad47d..0000000--- a/apps/api/workflows/sync-docs/steps/prepare-workspace.ts+++ /dev/null@@ -1,21 +0,0 @@-import { mkdir, rm } from 'node:fs/promises'-import { tmpdir } from 'node:os'-import { resolve } from 'pathe'-import { getLogger } from '@savoir/logger'--export async function prepareWorkspace(reset: boolean): Promise<string> {- 'use step'-- const logger = getLogger()- const syncDir = resolve(tmpdir(), 'savoir-sync', Date.now().toString())- await mkdir(syncDir, { recursive: true })-- if (reset) {- await rm(resolve(syncDir, 'docs'), { recursive: true, force: true })- }-- await mkdir(resolve(syncDir, 'docs'), { recursive: true })- logger.log('sync', `Workspace ready: ${syncDir}`)-- return syncDir-}diff --git a/apps/api/workflows/sync-docs/steps/push-snapshot.ts b/apps/api/workflows/sync-docs/steps/push-snapshot.tsdeleted file mode 100644index a0b89ff..0000000--- a/apps/api/workflows/sync-docs/steps/push-snapshot.ts+++ /dev/null@@ -1,34 +0,0 @@-import { getLogger } from '@savoir/logger'-import type { SyncConfig } from '../utils/index.js'-import { pushToSnapshot } from '../utils/index.js'--export async function pushToSnapshotStep(- syncDir: string,- config: SyncConfig,- successCount: number,- totalFiles: number-): Promise<{ success: boolean; commitSha?: string; error?: string }> {- 'use step'-- const logger = getLogger()- logger.log('sync', `Pushing to ${config.snapshotRepo}...`)-- const result = await pushToSnapshot(syncDir, {- repo: config.snapshotRepo,- branch: config.snapshotBranch || 'main',- token: config.githubToken,- message: `chore: sync ${successCount} sources (${totalFiles} files)`,- })-- if (result.success) {- logger.log('sync', `Pushed ${result.filesChanged} files, commit: ${result.commitSha}`)- } else {- throw new Error(`Push failed: ${result.error}`)- }-- return {- success: result.success,- commitSha: result.commitSha,- error: result.error,- }-}diff --git a/apps/api/workflows/sync-docs/steps/sandbox-sync.ts b/apps/api/workflows/sync-docs/steps/sandbox-sync.tsnew file mode 100644index 0000000..af1977c--- /dev/null+++ b/apps/api/workflows/sync-docs/steps/sandbox-sync.ts@@ -0,0 +1,184 @@+import { Sandbox } from '@vercel/sandbox'+import { getLogger } from '@savoir/logger'+import type { GitHubSource, SyncResult } from '@savoir/config'++interface SyncInSandboxResult {+ snapshotId: string+ results: SyncResult[]+}++/**+ * Sync all sources inside a sandbox and return the snapshot ID.+ *+ * This is a single step because the Sandbox object is not serializable+ * and cannot be passed between workflow steps.+ */+export async function syncAllSourcesInSandbox(+ sources: GitHubSource[],+ snapshotRepo: string,+ snapshotBranch: string,+ githubToken?: string,+): Promise<SyncInSandboxResult> {+ 'use step'++ const logger = getLogger()+ const repoUrl = `https://github.com/${snapshotRepo}.git`++ logger.log('sync', `Creating sandbox from ${snapshotRepo}#${snapshotBranch}`)++ // Create sandbox from snapshot repo+ const source: {+ type: 'git'+ url: string+ revision: string+ username?: string+ password?: string+ } = {+ type: 'git',+ url: repoUrl,+ revision: snapshotBranch,+ }++ if (githubToken) {+ source.username = 'x-access-token'+ source.password = githubToken+ }++ const sandbox = await Sandbox.create({+ source,+ timeout: 10 * 60 * 1000, // 10 minutes+ runtime: 'node24',+ })++ logger.log('sync', `Sandbox created: ${sandbox.sandboxId}`)++ // Sync each source+ const results: SyncResult[] = []++ for (const src of sources) {+ const result = await syncSource(sandbox, src, logger)+ results.push(result)+ }++ // Take snapshot+ logger.log('sync', 'Taking snapshot...')+ const snapshot = await sandbox.snapshot()+ logger.log('sync', `Snapshot created: ${snapshot.snapshotId}`)++ return {+ snapshotId: snapshot.snapshotId,+ results,+ }+}++/**+ * Sync a single source inside the sandbox (internal function, not a step)+ */+async function syncSource(+ sandbox: Sandbox,+ source: GitHubSource,+ logger: ReturnType<typeof getLogger>,+): Promise<SyncResult> {+ const outputPath = source.outputPath || source.id+ const targetDir = `/vercel/sandbox/docs/${outputPath}`++ logger.log('sync', `Syncing ${source.id} to ${targetDir}`)++ try {+ // Create target directory+ await sandbox.runCommand({+ cmd: 'mkdir',+ args: ['-p', targetDir],+ cwd: '/vercel/sandbox',+ })++ if (source.readmeOnly) {+ // Just fetch the README+ const readmeUrl = `https://raw.githubusercontent.com/${source.repo}/${source.branch}/README.md`+ const result = await sandbox.runCommand({+ cmd: 'curl',+ args: ['-sL', '-o', `${targetDir}/README.md`, readmeUrl],+ cwd: '/vercel/sandbox',+ })++ if (result.exitCode !== 0) {+ throw new Error(`Failed to fetch README: ${await result.stderr()}`)+ }++ return { sourceId: source.id, success: true, fileCount: 1 }+ }++ // Clone the content path from the repo+ const contentPath = source.contentPath || ''+ const tempDir = `/tmp/sync-${source.id}-${Date.now()}`++ // Clone repo to temp dir (sparse checkout for specific path)+ const cloneResult = await sandbox.runCommand({+ cmd: 'sh',+ args: [+ '-c',+ [+ `git clone --depth 1 --single-branch --branch ${source.branch}`,+ `--filter=blob:none --sparse`,+ `https://github.com/${source.repo}.git ${tempDir}`,+ `&& cd ${tempDir}`,+ `&& git sparse-checkout set ${contentPath || '.'}`,+ ].join(' '),+ ],+ cwd: '/vercel/sandbox',+ })++ if (cloneResult.exitCode !== 0) {+ const stderr = await cloneResult.stderr()+ throw new Error(`Git clone failed: ${stderr}`)+ }++ // Copy content to target dir+ const sourcePath = contentPath ? `${tempDir}/${contentPath}` : tempDir+ await sandbox.runCommand({+ cmd: 'sh',+ args: ['-c', `cp -r ${sourcePath}/* ${targetDir}/ 2>/dev/null || cp -r ${sourcePath}/. ${targetDir}/`],+ cwd: '/vercel/sandbox',+ })++ // Clean up non-doc files (keep only .md, .mdx, .yml, .yaml, .json)+ await sandbox.runCommand({+ cmd: 'sh',+ args: [+ '-c',+ `find ${targetDir} -type f ! \\( -name "*.md" -o -name "*.mdx" -o -name "*.yml" -o -name "*.yaml" -o -name "*.json" \\) -delete`,+ ],+ cwd: '/vercel/sandbox',+ })++ // Remove empty directories+ await sandbox.runCommand({+ cmd: 'sh',+ args: ['-c', `find ${targetDir} -type d -empty -delete`],+ cwd: '/vercel/sandbox',+ })++ // Cleanup temp dir+ await sandbox.runCommand({+ cmd: 'rm',+ args: ['-rf', tempDir],+ cwd: '/vercel/sandbox',+ })++ // Count files+ const countResult = await sandbox.runCommand({+ cmd: 'sh',+ args: ['-c', `find ${targetDir} -type f -name "*.md" -o -name "*.mdx" | wc -l`],+ cwd: '/vercel/sandbox',+ })++ const fileCount = parseInt((await countResult.stdout()).trim()) || 0++ logger.log('sync', `${source.id}: synced ${fileCount} files`)+ return { sourceId: source.id, success: true, fileCount }+ } catch (error) {+ const errorMessage = error instanceof Error ? error.message : String(error)+ logger.log('sync', `${source.id}: failed - ${errorMessage}`)+ return { sourceId: source.id, success: false, fileCount: 0, error: errorMessage }+ }+}diff --git a/apps/api/workflows/sync-docs/steps/sync-source.ts b/apps/api/workflows/sync-docs/steps/sync-source.tsdeleted file mode 100644index 104601d..0000000--- a/apps/api/workflows/sync-docs/steps/sync-source.ts+++ /dev/null@@ -1,28 +0,0 @@-import { getLogger } from '@savoir/logger'-import type { GitHubSource, SyncResult } from '../utils/index.js'-import { syncGitHubSource } from '../utils/index.js'--function formatDuration(ms: number): string {- return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(2)}s`-}--export async function syncSingleSource(- source: GitHubSource,- syncDir: string-): Promise<SyncResult> {- 'use step'-- const logger = getLogger()- logger.log('sync', `Syncing ${source.id}...`)-- const result = await syncGitHubSource(source, syncDir)-- if (result.success) {- logger.log('sync', `${source.id}: ${result.fileCount} files in ${formatDuration(result.duration ?? 0)}`)- } else {- logger.error({ source: source.id, error: result.error, message: 'Sync failed' })- throw new Error(`Failed to sync ${source.id}: ${result.error}`)- }-- return result-}diff --git a/apps/api/workflows/sync-docs/steps/trigger-snapshot.ts b/apps/api/workflows/sync-docs/steps/trigger-snapshot.tsdeleted file mode 100644index 2d65c12..0000000--- a/apps/api/workflows/sync-docs/steps/trigger-snapshot.ts+++ /dev/null@@ -1,23 +0,0 @@-import { getLogger } from '@savoir/logger'-import { start } from 'workflow/api'-import { createSnapshot } from '../../create-snapshot/index.js'--interface SnapshotConfig {- // GitHub auth for private repos- githubToken?: string- // Snapshot repo config- snapshotRepo: string- snapshotBranch: string-}--export async function triggerSnapshotStep(config: SnapshotConfig): Promise<void> {- 'use step'-- const logger = getLogger()- logger.log('sync', 'Triggering snapshot creation...')-- // Start the snapshot workflow asynchronously- await start(createSnapshot, [config])-- logger.log('sync', 'Snapshot workflow started')-}diff --git a/apps/api/workflows/sync-docs/utils/github.ts b/apps/api/workflows/sync-docs/utils/github.tsdeleted file mode 100644index d06ae62..0000000--- a/apps/api/workflows/sync-docs/utils/github.ts+++ /dev/null@@ -1,181 +0,0 @@-import { mkdir, rm, writeFile, readdir, stat, unlink, rmdir } from 'node:fs/promises'-import { resolve, join, extname } from 'pathe'-import { downloadTemplate } from 'giget'-import type { GitHubSource, SyncResult, ContentFile } from '@savoir/config'--const ALLOWED_EXTENSIONS = new Set(['.md', '.mdx', '.yml', '.yaml', '.json'])--const EXCLUDED_FILES = new Set([- 'package-lock.json',- 'yarn.lock',- 'pnpm-lock.yaml',- 'bun.lockb',- 'composer.lock',- 'Gemfile.lock',- 'Cargo.lock',- 'Pipfile.lock',- 'poetry.lock',- 'uv.lock',- 'go.sum',-])--function isAllowedFile(filename: string): boolean {- if (EXCLUDED_FILES.has(filename)) return false- return ALLOWED_EXTENSIONS.has(extname(filename).toLowerCase())-}--export async function cleanupNonDocFiles(dir: string): Promise<number> {- let mdCount = 0-- async function processDir(currentDir: string): Promise<void> {- let entries: string[]- try {- entries = await readdir(currentDir)- } catch {- return- }-- for (const entry of entries) {- const fullPath = join(currentDir, entry)- let stats- try {- stats = await stat(fullPath)- } catch {- continue- }-- if (stats.isDirectory()) {- await processDir(fullPath)- try {- const remaining = await readdir(fullPath)- if (remaining.length === 0) await rmdir(fullPath)- } catch {- // Ignore cleanup errors- }- } else if (stats.isFile()) {- if (isAllowedFile(entry)) {- if (entry.endsWith('.md') || entry.endsWith('.mdx')) mdCount++- } else {- try {- await unlink(fullPath)- } catch {- // Ignore delete errors- }- }- }- }- }-- await processDir(dir)- return mdCount-}--export async function fetchReadme(repo: string, branch: string): Promise<string> {- const response = await fetch(- `https://raw.githubusercontent.com/${repo}/${branch}/README.md`- )- if (!response.ok) {- throw new Error(`Failed to fetch README from ${repo}: ${response.statusText}`)- }- return response.text()-}--export async function syncGitHubSource(- source: GitHubSource,- outputDir: string-): Promise<SyncResult> {- const startTime = Date.now()- const outputFolder = source.outputPath || source.id- const targetDir = resolve(outputDir, 'docs', outputFolder)-- try {- await mkdir(targetDir, { recursive: true })-- if (source.readmeOnly) {- const content = await fetchReadme(source.repo, source.branch)- await writeFile(resolve(targetDir, 'README.md'), content, 'utf-8')- return {- sourceId: source.id,- success: true,- fileCount: 1,- duration: Date.now() - startTime,- }- }-- const template = `gh:${source.repo}/${source.contentPath}#${source.branch}`- await downloadTemplate(template, { dir: targetDir, force: true })-- const mdCount = await cleanupNonDocFiles(targetDir)-- if (source.additionalSyncs) {- for (const additional of source.additionalSyncs) {- const additionalTemplate = `gh:${additional.repo}/${additional.contentPath}#${additional.branch}`- try {- await downloadTemplate(additionalTemplate, { dir: targetDir, force: true })- await cleanupNonDocFiles(targetDir)- } catch (error) {- console.warn(`[${source.id}] Additional sync failed for ${additional.repo}:`, error)- }- }- }-- return {- sourceId: source.id,- success: true,- fileCount: mdCount,- duration: Date.now() - startTime,- }- } catch (error) {- return {- sourceId: source.id,- success: false,- error: error instanceof Error ? error.message : String(error),- duration: Date.now() - startTime,- }- }-}--export async function resetSourceDir(source: GitHubSource, outputDir: string): Promise<void> {- const targetDir = resolve(outputDir, 'docs', source.outputPath || source.id)- try {- await rm(targetDir, { recursive: true, force: true })- } catch {- // Ignore if directory doesn't exist- }-}--export async function collectFiles(dir: string, basePath = ''): Promise<ContentFile[]> {- const files: ContentFile[] = []-- async function processDir(currentDir: string, relativePath: string): Promise<void> {- let entries: string[]- try {- entries = await readdir(currentDir)- } catch {- return- }-- for (const entry of entries) {- const fullPath = join(currentDir, entry)- const relPath = relativePath ? `${relativePath}/${entry}` : entry-- let stats- try {- stats = await stat(fullPath)- } catch {- continue- }-- if (stats.isDirectory()) {- await processDir(fullPath, relPath)- } else if (stats.isFile() && isAllowedFile(entry)) {- const { readFile } = await import('node:fs/promises')- const content = await readFile(fullPath, 'utf-8')- files.push({ path: relPath, content })- }- }- }-- await processDir(dir, basePath)- return files-}diff --git a/apps/api/workflows/sync-docs/utils/index.ts b/apps/api/workflows/sync-docs/utils/index.tsindex 3ef91b1..741b5d5 100644--- a/apps/api/workflows/sync-docs/utils/index.ts+++ b/apps/api/workflows/sync-docs/utils/index.ts@@ -19,15 +19,3 @@ export {getSourceById,getSourcesByType,} from '@savoir/config'--// GitHub sync functions-export {- syncGitHubSource,- resetSourceDir,- cleanupNonDocFiles,- fetchReadme,- collectFiles,-} from './github.js'--// Snapshot-export { pushToSnapshot } from './snapshot.js'diff --git a/apps/api/workflows/sync-docs/utils/snapshot.ts b/apps/api/workflows/sync-docs/utils/snapshot.tsdeleted file mode 100644index 022531d..0000000--- a/apps/api/workflows/sync-docs/utils/snapshot.ts+++ /dev/null@@ -1,202 +0,0 @@-import { readdir, stat, readFile } from 'node:fs/promises'-import { join } from 'pathe'-import type { PushResult } from '@savoir/config'--interface GitHubTreeItem {- path: string- mode: '100644' | '100755' | '040000' | '160000' | '120000'- type: 'blob' | 'tree' | 'commit'- sha?: string- content?: string-}--interface GitHubCommit {- sha: string- tree: { sha: string }-}--async function collectAllFiles(- dir: string,- basePath = ''-): Promise<Array<{ path: string; content: string }>> {- const files: Array<{ path: string; content: string }> = []-- async function processDir(currentDir: string, relativePath: string): Promise<void> {- let entries: string[]- try {- entries = await readdir(currentDir)- } catch {- return- }-- for (const entry of entries) {- const fullPath = join(currentDir, entry)- const relPath = relativePath ? `${relativePath}/${entry}` : entry-- let stats- try {- stats = await stat(fullPath)- } catch {- continue- }-- if (stats.isDirectory()) {- await processDir(fullPath, relPath)- } else if (stats.isFile()) {- try {- const content = await readFile(fullPath, 'utf-8')- files.push({ path: relPath, content })- } catch {- // Skip unreadable files- }- }- }- }-- await processDir(dir, basePath)- return files-}--export async function pushToSnapshot(- contentDir: string,- options: {- repo: string- branch: string- token: string- message?: string- }-): Promise<PushResult> {- const { repo, branch, token, message = 'chore: sync content' } = options- const apiBase = 'https://api.github.com'-- const headers = {- Authorization: `Bearer ${token}`,- Accept: 'application/vnd.github.v3+json',- 'Content-Type': 'application/json',- 'X-GitHub-Api-Version': '2022-11-28',- }-- try {- const files = await collectAllFiles(contentDir)-- if (files.length === 0) {- return { success: false, error: 'No files to push' }- }-- let baseSha: string | null = null- let baseTreeSha: string | null = null-- try {- const refResponse = await fetch(- `${apiBase}/repos/${repo}/git/refs/heads/${branch}`,- { headers }- )-- if (refResponse.ok) {- const refData = await refResponse.json()- baseSha = refData.object.sha-- const commitResponse = await fetch(- `${apiBase}/repos/${repo}/git/commits/${baseSha}`,- { headers }- )- if (commitResponse.ok) {- const commitData: GitHubCommit = await commitResponse.json()- baseTreeSha = commitData.tree.sha- }- }- } catch {- // Branch doesn't exist yet, will create new- }-- const treeItems: GitHubTreeItem[] = []-- for (const file of files) {- if (file.content.length < 1024 * 100) {- treeItems.push({- path: file.path,- mode: '100644',- type: 'blob',- content: file.content,- })- } else {- const blobResponse = await fetch(`${apiBase}/repos/${repo}/git/blobs`, {- method: 'POST',- headers,- body: JSON.stringify({- content: Buffer.from(file.content).toString('base64'),- encoding: 'base64',- }),- })-- if (!blobResponse.ok) {- throw new Error(`Failed to create blob for ${file.path}`)- }-- const blobData = await blobResponse.json()- treeItems.push({- path: file.path,- mode: '100644',- type: 'blob',- sha: blobData.sha,- })- }- }-- const treeResponse = await fetch(`${apiBase}/repos/${repo}/git/trees`, {- method: 'POST',- headers,- body: JSON.stringify({ base_tree: baseTreeSha, tree: treeItems }),- })-- if (!treeResponse.ok) {- throw new Error(`Failed to create tree: ${await treeResponse.text()}`)- }-- const treeData = await treeResponse.json()-- const commitPayload: Record<string, unknown> = { message, tree: treeData.sha }- if (baseSha) commitPayload.parents = [baseSha]-- const commitResponse = await fetch(`${apiBase}/repos/${repo}/git/commits`, {- method: 'POST',- headers,- body: JSON.stringify(commitPayload),- })-- if (!commitResponse.ok) {- throw new Error(`Failed to create commit: ${await commitResponse.text()}`)- }-- const commitData = await commitResponse.json()-- const refPath = baseSha- ? `${apiBase}/repos/${repo}/git/refs/heads/${branch}`- : `${apiBase}/repos/${repo}/git/refs`-- const refPayload = baseSha- ? { sha: commitData.sha, force: false }- : { ref: `refs/heads/${branch}`, sha: commitData.sha }-- const refUpdateResponse = await fetch(refPath, {- method: baseSha ? 'PATCH' : 'POST',- headers,- body: JSON.stringify(refPayload),- })-- if (!refUpdateResponse.ok) {- throw new Error(`Failed to update ref: ${await refUpdateResponse.text()}`)- }-- return {- success: true,- commitSha: commitData.sha,- filesChanged: files.length,- }- } catch (error) {- return {- success: false,- error: error instanceof Error ? error.message : String(error),- }- }-}diff --git a/apps/api/workflows/sync-docs/workflow.ts b/apps/api/workflows/sync-docs/workflow.tsindex a5d602e..00b29cd 100644--- a/apps/api/workflows/sync-docs/workflow.ts+++ b/apps/api/workflows/sync-docs/workflow.ts@@ -1,93 +1,91 @@+import { useStorage } from 'nitro/storage'import { getLogger } from '@savoir/logger'import { FatalError } from 'workflow'-import type { SyncResult, SyncConfig, SyncOptions } from './utils/index.js'+import type { SnapshotMetadata } from '../../lib/sandbox/types.js'+import { KV_KEYS } from '../../lib/sandbox/types.js'+import type { SyncConfig, SyncOptions, SyncResult } from './utils/index.js'import {- prepareWorkspace,getSourcesToSync,- syncSingleSource,- pushToSnapshotStep,- cleanupWorkspace,- triggerSnapshotStep,+ syncAllSourcesInSandbox,} from './steps/index.js'interface SyncWorkflowResult {success: boolean+ snapshotId?: stringsummary: {total: numbersuccess: numberfailed: numberfiles: number}- push?: { success: boolean; commitSha?: string; error?: string } | nullresults: SyncResult[]}/**- * Workflow: Sync documentation from all configured sources.+ * Workflow: Sync documentation using Vercel Sandbox.+ *+ * All filesystem operations run inside a Vercel Sandbox,+ * making it compatible with serverless environments.*/export async function syncDocumentation(config: SyncConfig,- options: SyncOptions = {}+ options: SyncOptions = {},): Promise<SyncWorkflowResult> {'use workflow'- const { reset = false, push = true, sourceFilter } = options+ const { sourceFilter } = options+ const logger = getLogger()- if (!config.githubToken) {- throw new FatalError('GITHUB_TOKEN is not configured')- }if (!config.snapshotRepo) {throw new FatalError('GITHUB_SNAPSHOT_REPO is not configured')}- const syncDir = await prepareWorkspace(reset)+ // Step 1: Get sources to syncconst sources = await getSourcesToSync(sourceFilter)if (sources.length === 0) {- await cleanupWorkspace(syncDir)throw new FatalError('No sources to sync')}- const results: SyncResult[] = []- for (const source of sources) {- const result = await syncSingleSource(source, syncDir)- results.push(result)- }+ logger.log('sync', `Syncing ${sources.length} sources...`)- const successCount = results.filter((r) => r.success).length- const failCount = results.filter((r) => !r.success).length- const totalFiles = results.reduce((sum, r) => sum + (r.fileCount || 0), 0)+ // Step 2: Sync all sources in sandbox and get snapshot+ // This is a single step because the Sandbox object is not serializable+ const { snapshotId, results } = await syncAllSourcesInSandbox(+ sources,+ config.snapshotRepo,+ config.snapshotBranch || 'main',+ config.githubToken,+ )- let pushResult = null- if (push && successCount > 0) {- pushResult = await pushToSnapshotStep(syncDir, config, successCount, totalFiles)+ const successCount = results.filter(r => r.success).length+ const failCount = results.filter(r => !r.success).length+ const totalFiles = results.reduce((sum, r) => sum + (r.fileCount || 0), 0)- // Trigger sandbox snapshot creation after successful push- if (pushResult.success) {- await triggerSnapshotStep({- githubToken: config.githubToken, // For private repos- snapshotRepo: config.snapshotRepo,- snapshotBranch: config.snapshotBranch,- })- }+ // Step 3: Store snapshot metadata in KV+ const metadata: SnapshotMetadata = {+ snapshotId,+ createdAt: Date.now(),+ sourceRepo: config.snapshotRepo,}- await cleanupWorkspace(syncDir)+ const kv = useStorage('kv')+ await kv.setItem(KV_KEYS.CURRENT_SNAPSHOT, metadata)+ logger.log('sync', 'Snapshot metadata stored in KV')// Log summary- const logger = getLogger()const status = failCount === 0 ? '✓' : '✗'logger.log('sync', `${status} Done: ${successCount}/${sources.length} sources, ${totalFiles} files`)return {- success: failCount === 0 && (!push || pushResult?.success !== false),+ success: failCount === 0,+ snapshotId,summary: {total: sources.length,success: successCount,failed: failCount,files: totalFiles,},- push: pushResult,results,}}diff --git a/savoir.config.ts b/savoir.config.tsindex 793b6ff..75b86f8 100644--- a/savoir.config.ts+++ b/savoir.config.ts@@ -8,15 +8,13 @@ export default defineConfig({id: 'nuxt',label: 'Nuxt',repo: 'nuxt/nuxt',- additionalSyncs: [- { repo: 'nuxt/nuxt.com', contentPath: 'content' },- ],+ additionalSyncs: [{ repo: 'nuxt/nuxt.com', contentPath: 'content' }],},{id: 'nitro',label: 'Nitro',repo: 'nitrojs/nitro',- branch: 'v3',+ branch: 'main',},// Nuxt Modules