Skip to content

Open source file-system and knowledge based agent template. Build AI agents that stay up to date with your knowledge base

use sandboxes in workflow

Hugo Richard656b4f530620
diff --git a/apps/api/routes/api/sync/[source].post.ts b/apps/api/routes/api/sync/[source].post.ts
index da296d0..8ce35d9 100644
--- a/apps/api/routes/api/sync/[source].post.ts
+++ b/apps/api/routes/api/sync/[source].post.ts
@@ -1,5 +1,5 @@
import { start } from 'workflow/api'
-import { defineHandler, readValidatedBody, getValidatedRouterParams } from 'nitro/h3'
+import { defineHandler, getValidatedRouterParams } from 'nitro/h3'
import { useRuntimeConfig } from 'nitro/runtime-config'
import { z } from 'zod'
import { syncDocumentation } from '~/workflows/sync-docs'
@@ -8,27 +8,15 @@ const paramsSchema = z.object({
source: z.string().min(1),
})
-const bodySchema = z
- .object({
- reset: z.boolean().default(false),
- push: z.boolean().default(true),
- })
- .optional()
-
/**
* POST /api/sync/:source
- * Sync a specific source.
+ * Sync a specific source using Vercel Sandbox.
*
* Params:
* - source: Source ID to sync
- *
- * Body (optional):
- * - reset: boolean - Clear content before sync (default: false)
- * - push: boolean - Push to snapshot repo after sync (default: true)
*/
export default defineHandler(async (event) => {
const { source } = await getValidatedRouterParams(event, paramsSchema.parse)
- const body = await readValidatedBody(event, data => bodySchema.parse(data))
const config = useRuntimeConfig()
const syncConfig = {
@@ -38,8 +26,6 @@ export default defineHandler(async (event) => {
}
const options = {
- reset: body?.reset ?? false,
- push: body?.push ?? true,
sourceFilter: source,
}
@@ -49,6 +35,5 @@ export default defineHandler(async (event) => {
status: 'started',
message: `Sync workflow started for source "${source}". Use \`pnpm workflow:web\` to monitor.`,
source,
- options,
}
})
diff --git a/apps/api/routes/api/sync/index.post.ts b/apps/api/routes/api/sync/index.post.ts
index d7883ac..e773f68 100644
--- a/apps/api/routes/api/sync/index.post.ts
+++ b/apps/api/routes/api/sync/index.post.ts
@@ -6,18 +6,16 @@ import { syncDocumentation } from '~/workflows/sync-docs'
const bodySchema = z
.object({
- reset: z.boolean().default(false),
- push: z.boolean().default(true),
+ sourceFilter: z.string().optional(),
})
.optional()
/**
* POST /api/sync
- * Sync all sources.
+ * Sync all sources using Vercel Sandbox.
*
* Body (optional):
- * - reset: boolean - Clear all content before sync (default: false)
- * - push: boolean - Push to snapshot repo after sync (default: true)
+ * - sourceFilter: string - Only sync a specific source
*/
export default defineHandler(async (event) => {
const body = await readValidatedBody(event, data => bodySchema.parse(data))
@@ -30,8 +28,7 @@ export default defineHandler(async (event) => {
}
const options = {
- reset: body?.reset ?? false,
- push: body?.push ?? true,
+ sourceFilter: body?.sourceFilter,
}
await start(syncDocumentation, [syncConfig, options])
@@ -39,6 +36,5 @@ export default defineHandler(async (event) => {
return {
status: 'started',
message: 'Sync workflow started. Use `pnpm workflow:web` to monitor.',
- options,
}
})
diff --git a/apps/api/workflows/create-snapshot/workflow.ts b/apps/api/workflows/create-snapshot/workflow.ts
index 4321dfd..53cfb31 100644
--- a/apps/api/workflows/create-snapshot/workflow.ts
+++ b/apps/api/workflows/create-snapshot/workflow.ts
@@ -87,7 +87,7 @@ async function storeSnapshotMetadataStep(
const isProduction = !!(process.env.KV_REST_API_URL && process.env.KV_REST_API_TOKEN)
if (isProduction) {
- // Use Upstash Redis in production
+ // eslint-disable-next-line
const { Redis } = await import('@upstash/redis')
const redis = new Redis({
url: process.env.KV_REST_API_URL!,
@@ -95,8 +95,7 @@ async function storeSnapshotMetadataStep(
})
await redis.set('snapshot:current', metadata)
logger.log('snapshot', 'Snapshot metadata stored in Upstash Redis')
- }
- else {
+ } else {
// Use filesystem in development
const kvDir = join(process.cwd(), '.data', 'kv')
const filePath = join(kvDir, 'snapshot:current.json')
diff --git a/apps/api/workflows/sync-docs/steps/cleanup.ts b/apps/api/workflows/sync-docs/steps/cleanup.ts
deleted file mode 100644
index 911d9d2..0000000
--- a/apps/api/workflows/sync-docs/steps/cleanup.ts
+++ /dev/null
@@ -1,15 +0,0 @@
-import { rm } from 'node:fs/promises'
-import { getLogger } from '@savoir/logger'
-
-export async function cleanupWorkspace(syncDir: string): Promise<void> {
- 'use step'
-
- const logger = getLogger()
-
- try {
- await rm(syncDir, { recursive: true, force: true })
- logger.log('sync', 'Workspace cleaned up')
- } catch {
- logger.log('sync', 'Failed to cleanup workspace')
- }
-}
diff --git a/apps/api/workflows/sync-docs/steps/index.ts b/apps/api/workflows/sync-docs/steps/index.ts
index 0152c7e..8d61685 100644
--- a/apps/api/workflows/sync-docs/steps/index.ts
+++ b/apps/api/workflows/sync-docs/steps/index.ts
@@ -1,6 +1,4 @@
-export { prepareWorkspace } from './prepare-workspace.js'
export { getSourcesToSync } from './get-sources.js'
-export { syncSingleSource } from './sync-source.js'
-export { pushToSnapshotStep } from './push-snapshot.js'
-export { cleanupWorkspace } from './cleanup.js'
-export { triggerSnapshotStep } from './trigger-snapshot.js'
+
+// Sandbox-based step (single step to avoid serialization issues)
+export { syncAllSourcesInSandbox } from './sandbox-sync.js'
diff --git a/apps/api/workflows/sync-docs/steps/prepare-workspace.ts b/apps/api/workflows/sync-docs/steps/prepare-workspace.ts
deleted file mode 100644
index 09ad47d..0000000
--- a/apps/api/workflows/sync-docs/steps/prepare-workspace.ts
+++ /dev/null
@@ -1,21 +0,0 @@
-import { mkdir, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { resolve } from 'pathe'
-import { getLogger } from '@savoir/logger'
-
-export async function prepareWorkspace(reset: boolean): Promise<string> {
- 'use step'
-
- const logger = getLogger()
- const syncDir = resolve(tmpdir(), 'savoir-sync', Date.now().toString())
- await mkdir(syncDir, { recursive: true })
-
- if (reset) {
- await rm(resolve(syncDir, 'docs'), { recursive: true, force: true })
- }
-
- await mkdir(resolve(syncDir, 'docs'), { recursive: true })
- logger.log('sync', `Workspace ready: ${syncDir}`)
-
- return syncDir
-}
diff --git a/apps/api/workflows/sync-docs/steps/push-snapshot.ts b/apps/api/workflows/sync-docs/steps/push-snapshot.ts
deleted file mode 100644
index a0b89ff..0000000
--- a/apps/api/workflows/sync-docs/steps/push-snapshot.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-import { getLogger } from '@savoir/logger'
-import type { SyncConfig } from '../utils/index.js'
-import { pushToSnapshot } from '../utils/index.js'
-
-export async function pushToSnapshotStep(
- syncDir: string,
- config: SyncConfig,
- successCount: number,
- totalFiles: number
-): Promise<{ success: boolean; commitSha?: string; error?: string }> {
- 'use step'
-
- const logger = getLogger()
- logger.log('sync', `Pushing to ${config.snapshotRepo}...`)
-
- const result = await pushToSnapshot(syncDir, {
- repo: config.snapshotRepo,
- branch: config.snapshotBranch || 'main',
- token: config.githubToken,
- message: `chore: sync ${successCount} sources (${totalFiles} files)`,
- })
-
- if (result.success) {
- logger.log('sync', `Pushed ${result.filesChanged} files, commit: ${result.commitSha}`)
- } else {
- throw new Error(`Push failed: ${result.error}`)
- }
-
- return {
- success: result.success,
- commitSha: result.commitSha,
- error: result.error,
- }
-}
diff --git a/apps/api/workflows/sync-docs/steps/sandbox-sync.ts b/apps/api/workflows/sync-docs/steps/sandbox-sync.ts
new file mode 100644
index 0000000..af1977c
--- /dev/null
+++ b/apps/api/workflows/sync-docs/steps/sandbox-sync.ts
@@ -0,0 +1,184 @@
+import { Sandbox } from '@vercel/sandbox'
+import { getLogger } from '@savoir/logger'
+import type { GitHubSource, SyncResult } from '@savoir/config'
+
+interface SyncInSandboxResult {
+ snapshotId: string
+ results: SyncResult[]
+}
+
+/**
+ * Sync all sources inside a sandbox and return the snapshot ID.
+ *
+ * This is a single step because the Sandbox object is not serializable
+ * and cannot be passed between workflow steps.
+ */
+export async function syncAllSourcesInSandbox(
+ sources: GitHubSource[],
+ snapshotRepo: string,
+ snapshotBranch: string,
+ githubToken?: string,
+): Promise<SyncInSandboxResult> {
+ 'use step'
+
+ const logger = getLogger()
+ const repoUrl = `https://github.com/${snapshotRepo}.git`
+
+ logger.log('sync', `Creating sandbox from ${snapshotRepo}#${snapshotBranch}`)
+
+ // Create sandbox from snapshot repo
+ const source: {
+ type: 'git'
+ url: string
+ revision: string
+ username?: string
+ password?: string
+ } = {
+ type: 'git',
+ url: repoUrl,
+ revision: snapshotBranch,
+ }
+
+ if (githubToken) {
+ source.username = 'x-access-token'
+ source.password = githubToken
+ }
+
+ const sandbox = await Sandbox.create({
+ source,
+ timeout: 10 * 60 * 1000, // 10 minutes
+ runtime: 'node24',
+ })
+
+ logger.log('sync', `Sandbox created: ${sandbox.sandboxId}`)
+
+ // Sync each source
+ const results: SyncResult[] = []
+
+ for (const src of sources) {
+ const result = await syncSource(sandbox, src, logger)
+ results.push(result)
+ }
+
+ // Take snapshot
+ logger.log('sync', 'Taking snapshot...')
+ const snapshot = await sandbox.snapshot()
+ logger.log('sync', `Snapshot created: ${snapshot.snapshotId}`)
+
+ return {
+ snapshotId: snapshot.snapshotId,
+ results,
+ }
+}
+
+/**
+ * Sync a single source inside the sandbox (internal function, not a step)
+ */
+async function syncSource(
+ sandbox: Sandbox,
+ source: GitHubSource,
+ logger: ReturnType<typeof getLogger>,
+): Promise<SyncResult> {
+ const outputPath = source.outputPath || source.id
+ const targetDir = `/vercel/sandbox/docs/${outputPath}`
+
+ logger.log('sync', `Syncing ${source.id} to ${targetDir}`)
+
+ try {
+ // Create target directory
+ await sandbox.runCommand({
+ cmd: 'mkdir',
+ args: ['-p', targetDir],
+ cwd: '/vercel/sandbox',
+ })
+
+ if (source.readmeOnly) {
+ // Just fetch the README
+ const readmeUrl = `https://raw.githubusercontent.com/${source.repo}/${source.branch}/README.md`
+ const result = await sandbox.runCommand({
+ cmd: 'curl',
+ args: ['-sL', '-o', `${targetDir}/README.md`, readmeUrl],
+ cwd: '/vercel/sandbox',
+ })
+
+ if (result.exitCode !== 0) {
+ throw new Error(`Failed to fetch README: ${await result.stderr()}`)
+ }
+
+ return { sourceId: source.id, success: true, fileCount: 1 }
+ }
+
+ // Clone the content path from the repo
+ const contentPath = source.contentPath || ''
+ const tempDir = `/tmp/sync-${source.id}-${Date.now()}`
+
+ // Clone repo to temp dir (sparse checkout for specific path)
+ const cloneResult = await sandbox.runCommand({
+ cmd: 'sh',
+ args: [
+ '-c',
+ [
+ `git clone --depth 1 --single-branch --branch ${source.branch}`,
+ `--filter=blob:none --sparse`,
+ `https://github.com/${source.repo}.git ${tempDir}`,
+ `&& cd ${tempDir}`,
+ `&& git sparse-checkout set ${contentPath || '.'}`,
+ ].join(' '),
+ ],
+ cwd: '/vercel/sandbox',
+ })
+
+ if (cloneResult.exitCode !== 0) {
+ const stderr = await cloneResult.stderr()
+ throw new Error(`Git clone failed: ${stderr}`)
+ }
+
+ // Copy content to target dir
+ const sourcePath = contentPath ? `${tempDir}/${contentPath}` : tempDir
+ await sandbox.runCommand({
+ cmd: 'sh',
+ args: ['-c', `cp -r ${sourcePath}/* ${targetDir}/ 2>/dev/null || cp -r ${sourcePath}/. ${targetDir}/`],
+ cwd: '/vercel/sandbox',
+ })
+
+ // Clean up non-doc files (keep only .md, .mdx, .yml, .yaml, .json)
+ await sandbox.runCommand({
+ cmd: 'sh',
+ args: [
+ '-c',
+ `find ${targetDir} -type f ! \\( -name "*.md" -o -name "*.mdx" -o -name "*.yml" -o -name "*.yaml" -o -name "*.json" \\) -delete`,
+ ],
+ cwd: '/vercel/sandbox',
+ })
+
+ // Remove empty directories
+ await sandbox.runCommand({
+ cmd: 'sh',
+ args: ['-c', `find ${targetDir} -type d -empty -delete`],
+ cwd: '/vercel/sandbox',
+ })
+
+ // Cleanup temp dir
+ await sandbox.runCommand({
+ cmd: 'rm',
+ args: ['-rf', tempDir],
+ cwd: '/vercel/sandbox',
+ })
+
+ // Count files
+ const countResult = await sandbox.runCommand({
+ cmd: 'sh',
+ args: ['-c', `find ${targetDir} -type f -name "*.md" -o -name "*.mdx" | wc -l`],
+ cwd: '/vercel/sandbox',
+ })
+
+ const fileCount = parseInt((await countResult.stdout()).trim()) || 0
+
+ logger.log('sync', `${source.id}: synced ${fileCount} files`)
+ return { sourceId: source.id, success: true, fileCount }
+ } catch (error) {
+ const errorMessage = error instanceof Error ? error.message : String(error)
+ logger.log('sync', `${source.id}: failed - ${errorMessage}`)
+ return { sourceId: source.id, success: false, fileCount: 0, error: errorMessage }
+ }
+}
diff --git a/apps/api/workflows/sync-docs/steps/sync-source.ts b/apps/api/workflows/sync-docs/steps/sync-source.ts
deleted file mode 100644
index 104601d..0000000
--- a/apps/api/workflows/sync-docs/steps/sync-source.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import { getLogger } from '@savoir/logger'
-import type { GitHubSource, SyncResult } from '../utils/index.js'
-import { syncGitHubSource } from '../utils/index.js'
-
-function formatDuration(ms: number): string {
- return ms < 1000 ? `${Math.round(ms)}ms` : `${(ms / 1000).toFixed(2)}s`
-}
-
-export async function syncSingleSource(
- source: GitHubSource,
- syncDir: string
-): Promise<SyncResult> {
- 'use step'
-
- const logger = getLogger()
- logger.log('sync', `Syncing ${source.id}...`)
-
- const result = await syncGitHubSource(source, syncDir)
-
- if (result.success) {
- logger.log('sync', `${source.id}: ${result.fileCount} files in ${formatDuration(result.duration ?? 0)}`)
- } else {
- logger.error({ source: source.id, error: result.error, message: 'Sync failed' })
- throw new Error(`Failed to sync ${source.id}: ${result.error}`)
- }
-
- return result
-}
diff --git a/apps/api/workflows/sync-docs/steps/trigger-snapshot.ts b/apps/api/workflows/sync-docs/steps/trigger-snapshot.ts
deleted file mode 100644
index 2d65c12..0000000
--- a/apps/api/workflows/sync-docs/steps/trigger-snapshot.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import { getLogger } from '@savoir/logger'
-import { start } from 'workflow/api'
-import { createSnapshot } from '../../create-snapshot/index.js'
-
-interface SnapshotConfig {
- // GitHub auth for private repos
- githubToken?: string
- // Snapshot repo config
- snapshotRepo: string
- snapshotBranch: string
-}
-
-export async function triggerSnapshotStep(config: SnapshotConfig): Promise<void> {
- 'use step'
-
- const logger = getLogger()
- logger.log('sync', 'Triggering snapshot creation...')
-
- // Start the snapshot workflow asynchronously
- await start(createSnapshot, [config])
-
- logger.log('sync', 'Snapshot workflow started')
-}
diff --git a/apps/api/workflows/sync-docs/utils/github.ts b/apps/api/workflows/sync-docs/utils/github.ts
deleted file mode 100644
index d06ae62..0000000
--- a/apps/api/workflows/sync-docs/utils/github.ts
+++ /dev/null
@@ -1,181 +0,0 @@
-import { mkdir, rm, writeFile, readdir, stat, unlink, rmdir } from 'node:fs/promises'
-import { resolve, join, extname } from 'pathe'
-import { downloadTemplate } from 'giget'
-import type { GitHubSource, SyncResult, ContentFile } from '@savoir/config'
-
-const ALLOWED_EXTENSIONS = new Set(['.md', '.mdx', '.yml', '.yaml', '.json'])
-
-const EXCLUDED_FILES = new Set([
- 'package-lock.json',
- 'yarn.lock',
- 'pnpm-lock.yaml',
- 'bun.lockb',
- 'composer.lock',
- 'Gemfile.lock',
- 'Cargo.lock',
- 'Pipfile.lock',
- 'poetry.lock',
- 'uv.lock',
- 'go.sum',
-])
-
-function isAllowedFile(filename: string): boolean {
- if (EXCLUDED_FILES.has(filename)) return false
- return ALLOWED_EXTENSIONS.has(extname(filename).toLowerCase())
-}
-
-export async function cleanupNonDocFiles(dir: string): Promise<number> {
- let mdCount = 0
-
- async function processDir(currentDir: string): Promise<void> {
- let entries: string[]
- try {
- entries = await readdir(currentDir)
- } catch {
- return
- }
-
- for (const entry of entries) {
- const fullPath = join(currentDir, entry)
- let stats
- try {
- stats = await stat(fullPath)
- } catch {
- continue
- }
-
- if (stats.isDirectory()) {
- await processDir(fullPath)
- try {
- const remaining = await readdir(fullPath)
- if (remaining.length === 0) await rmdir(fullPath)
- } catch {
- // Ignore cleanup errors
- }
- } else if (stats.isFile()) {
- if (isAllowedFile(entry)) {
- if (entry.endsWith('.md') || entry.endsWith('.mdx')) mdCount++
- } else {
- try {
- await unlink(fullPath)
- } catch {
- // Ignore delete errors
- }
- }
- }
- }
- }
-
- await processDir(dir)
- return mdCount
-}
-
-export async function fetchReadme(repo: string, branch: string): Promise<string> {
- const response = await fetch(
- `https://raw.githubusercontent.com/${repo}/${branch}/README.md`
- )
- if (!response.ok) {
- throw new Error(`Failed to fetch README from ${repo}: ${response.statusText}`)
- }
- return response.text()
-}
-
-export async function syncGitHubSource(
- source: GitHubSource,
- outputDir: string
-): Promise<SyncResult> {
- const startTime = Date.now()
- const outputFolder = source.outputPath || source.id
- const targetDir = resolve(outputDir, 'docs', outputFolder)
-
- try {
- await mkdir(targetDir, { recursive: true })
-
- if (source.readmeOnly) {
- const content = await fetchReadme(source.repo, source.branch)
- await writeFile(resolve(targetDir, 'README.md'), content, 'utf-8')
- return {
- sourceId: source.id,
- success: true,
- fileCount: 1,
- duration: Date.now() - startTime,
- }
- }
-
- const template = `gh:${source.repo}/${source.contentPath}#${source.branch}`
- await downloadTemplate(template, { dir: targetDir, force: true })
-
- const mdCount = await cleanupNonDocFiles(targetDir)
-
- if (source.additionalSyncs) {
- for (const additional of source.additionalSyncs) {
- const additionalTemplate = `gh:${additional.repo}/${additional.contentPath}#${additional.branch}`
- try {
- await downloadTemplate(additionalTemplate, { dir: targetDir, force: true })
- await cleanupNonDocFiles(targetDir)
- } catch (error) {
- console.warn(`[${source.id}] Additional sync failed for ${additional.repo}:`, error)
- }
- }
- }
-
- return {
- sourceId: source.id,
- success: true,
- fileCount: mdCount,
- duration: Date.now() - startTime,
- }
- } catch (error) {
- return {
- sourceId: source.id,
- success: false,
- error: error instanceof Error ? error.message : String(error),
- duration: Date.now() - startTime,
- }
- }
-}
-
-export async function resetSourceDir(source: GitHubSource, outputDir: string): Promise<void> {
- const targetDir = resolve(outputDir, 'docs', source.outputPath || source.id)
- try {
- await rm(targetDir, { recursive: true, force: true })
- } catch {
- // Ignore if directory doesn't exist
- }
-}
-
-export async function collectFiles(dir: string, basePath = ''): Promise<ContentFile[]> {
- const files: ContentFile[] = []
-
- async function processDir(currentDir: string, relativePath: string): Promise<void> {
- let entries: string[]
- try {
- entries = await readdir(currentDir)
- } catch {
- return
- }
-
- for (const entry of entries) {
- const fullPath = join(currentDir, entry)
- const relPath = relativePath ? `${relativePath}/${entry}` : entry
-
- let stats
- try {
- stats = await stat(fullPath)
- } catch {
- continue
- }
-
- if (stats.isDirectory()) {
- await processDir(fullPath, relPath)
- } else if (stats.isFile() && isAllowedFile(entry)) {
- const { readFile } = await import('node:fs/promises')
- const content = await readFile(fullPath, 'utf-8')
- files.push({ path: relPath, content })
- }
- }
- }
-
- await processDir(dir, basePath)
- return files
-}
diff --git a/apps/api/workflows/sync-docs/utils/index.ts b/apps/api/workflows/sync-docs/utils/index.ts
index 3ef91b1..741b5d5 100644
--- a/apps/api/workflows/sync-docs/utils/index.ts
+++ b/apps/api/workflows/sync-docs/utils/index.ts
@@ -19,15 +19,3 @@ export {
getSourceById,
getSourcesByType,
} from '@savoir/config'
-
-// GitHub sync functions
-export {
- syncGitHubSource,
- resetSourceDir,
- cleanupNonDocFiles,
- fetchReadme,
- collectFiles,
-} from './github.js'
-
-// Snapshot
-export { pushToSnapshot } from './snapshot.js'
diff --git a/apps/api/workflows/sync-docs/utils/snapshot.ts b/apps/api/workflows/sync-docs/utils/snapshot.ts
deleted file mode 100644
index 022531d..0000000
--- a/apps/api/workflows/sync-docs/utils/snapshot.ts
+++ /dev/null
@@ -1,202 +0,0 @@
-import { readdir, stat, readFile } from 'node:fs/promises'
-import { join } from 'pathe'
-import type { PushResult } from '@savoir/config'
-
-interface GitHubTreeItem {
- path: string
- mode: '100644' | '100755' | '040000' | '160000' | '120000'
- type: 'blob' | 'tree' | 'commit'
- sha?: string
- content?: string
-}
-
-interface GitHubCommit {
- sha: string
- tree: { sha: string }
-}
-
-async function collectAllFiles(
- dir: string,
- basePath = ''
-): Promise<Array<{ path: string; content: string }>> {
- const files: Array<{ path: string; content: string }> = []
-
- async function processDir(currentDir: string, relativePath: string): Promise<void> {
- let entries: string[]
- try {
- entries = await readdir(currentDir)
- } catch {
- return
- }
-
- for (const entry of entries) {
- const fullPath = join(currentDir, entry)
- const relPath = relativePath ? `${relativePath}/${entry}` : entry
-
- let stats
- try {
- stats = await stat(fullPath)
- } catch {
- continue
- }
-
- if (stats.isDirectory()) {
- await processDir(fullPath, relPath)
- } else if (stats.isFile()) {
- try {
- const content = await readFile(fullPath, 'utf-8')
- files.push({ path: relPath, content })
- } catch {
- // Skip unreadable files
- }
- }
- }
- }
-
- await processDir(dir, basePath)
- return files
-}
-
-export async function pushToSnapshot(
- contentDir: string,
- options: {
- repo: string
- branch: string
- token: string
- message?: string
- }
-): Promise<PushResult> {
- const { repo, branch, token, message = 'chore: sync content' } = options
- const apiBase = 'https://api.github.com'
-
- const headers = {
- Authorization: `Bearer ${token}`,
- Accept: 'application/vnd.github.v3+json',
- 'Content-Type': 'application/json',
- 'X-GitHub-Api-Version': '2022-11-28',
- }
-
- try {
- const files = await collectAllFiles(contentDir)
-
- if (files.length === 0) {
- return { success: false, error: 'No files to push' }
- }
-
- let baseSha: string | null = null
- let baseTreeSha: string | null = null
-
- try {
- const refResponse = await fetch(
- `${apiBase}/repos/${repo}/git/refs/heads/${branch}`,
- { headers }
- )
-
- if (refResponse.ok) {
- const refData = await refResponse.json()
- baseSha = refData.object.sha
-
- const commitResponse = await fetch(
- `${apiBase}/repos/${repo}/git/commits/${baseSha}`,
- { headers }
- )
- if (commitResponse.ok) {
- const commitData: GitHubCommit = await commitResponse.json()
- baseTreeSha = commitData.tree.sha
- }
- }
- } catch {
- // Branch doesn't exist yet, will create new
- }
-
- const treeItems: GitHubTreeItem[] = []
-
- for (const file of files) {
- if (file.content.length < 1024 * 100) {
- treeItems.push({
- path: file.path,
- mode: '100644',
- type: 'blob',
- content: file.content,
- })
- } else {
- const blobResponse = await fetch(`${apiBase}/repos/${repo}/git/blobs`, {
- method: 'POST',
- headers,
- body: JSON.stringify({
- content: Buffer.from(file.content).toString('base64'),
- encoding: 'base64',
- }),
- })
-
- if (!blobResponse.ok) {
- throw new Error(`Failed to create blob for ${file.path}`)
- }
-
- const blobData = await blobResponse.json()
- treeItems.push({
- path: file.path,
- mode: '100644',
- type: 'blob',
- sha: blobData.sha,
- })
- }
- }
-
- const treeResponse = await fetch(`${apiBase}/repos/${repo}/git/trees`, {
- method: 'POST',
- headers,
- body: JSON.stringify({ base_tree: baseTreeSha, tree: treeItems }),
- })
-
- if (!treeResponse.ok) {
- throw new Error(`Failed to create tree: ${await treeResponse.text()}`)
- }
-
- const treeData = await treeResponse.json()
-
- const commitPayload: Record<string, unknown> = { message, tree: treeData.sha }
- if (baseSha) commitPayload.parents = [baseSha]
-
- const commitResponse = await fetch(`${apiBase}/repos/${repo}/git/commits`, {
- method: 'POST',
- headers,
- body: JSON.stringify(commitPayload),
- })
-
- if (!commitResponse.ok) {
- throw new Error(`Failed to create commit: ${await commitResponse.text()}`)
- }
-
- const commitData = await commitResponse.json()
-
- const refPath = baseSha
- ? `${apiBase}/repos/${repo}/git/refs/heads/${branch}`
- : `${apiBase}/repos/${repo}/git/refs`
-
- const refPayload = baseSha
- ? { sha: commitData.sha, force: false }
- : { ref: `refs/heads/${branch}`, sha: commitData.sha }
-
- const refUpdateResponse = await fetch(refPath, {
- method: baseSha ? 'PATCH' : 'POST',
- headers,
- body: JSON.stringify(refPayload),
- })
-
- if (!refUpdateResponse.ok) {
- throw new Error(`Failed to update ref: ${await refUpdateResponse.text()}`)
- }
-
- return {
- success: true,
- commitSha: commitData.sha,
- filesChanged: files.length,
- }
- } catch (error) {
- return {
- success: false,
- error: error instanceof Error ? error.message : String(error),
- }
- }
-}
diff --git a/apps/api/workflows/sync-docs/workflow.ts b/apps/api/workflows/sync-docs/workflow.ts
index a5d602e..00b29cd 100644
--- a/apps/api/workflows/sync-docs/workflow.ts
+++ b/apps/api/workflows/sync-docs/workflow.ts
@@ -1,93 +1,91 @@
+import { useStorage } from 'nitro/storage'
import { getLogger } from '@savoir/logger'
import { FatalError } from 'workflow'
-import type { SyncResult, SyncConfig, SyncOptions } from './utils/index.js'
+import type { SnapshotMetadata } from '../../lib/sandbox/types.js'
+import { KV_KEYS } from '../../lib/sandbox/types.js'
+import type { SyncConfig, SyncOptions, SyncResult } from './utils/index.js'
import {
- prepareWorkspace,
getSourcesToSync,
- syncSingleSource,
- pushToSnapshotStep,
- cleanupWorkspace,
- triggerSnapshotStep,
+ syncAllSourcesInSandbox,
} from './steps/index.js'
interface SyncWorkflowResult {
success: boolean
+ snapshotId?: string
summary: {
total: number
success: number
failed: number
files: number
}
- push?: { success: boolean; commitSha?: string; error?: string } | null
results: SyncResult[]
}
/**
- * Workflow: Sync documentation from all configured sources.
+ * Workflow: Sync documentation using Vercel Sandbox.
+ *
+ * All filesystem operations run inside a Vercel Sandbox,
+ * making it compatible with serverless environments.
*/
export async function syncDocumentation(
config: SyncConfig,
- options: SyncOptions = {}
+ options: SyncOptions = {},
): Promise<SyncWorkflowResult> {
'use workflow'
- const { reset = false, push = true, sourceFilter } = options
+ const { sourceFilter } = options
+ const logger = getLogger()
- if (!config.githubToken) {
- throw new FatalError('GITHUB_TOKEN is not configured')
- }
if (!config.snapshotRepo) {
throw new FatalError('GITHUB_SNAPSHOT_REPO is not configured')
}
- const syncDir = await prepareWorkspace(reset)
+ // Step 1: Get sources to sync
const sources = await getSourcesToSync(sourceFilter)
if (sources.length === 0) {
- await cleanupWorkspace(syncDir)
throw new FatalError('No sources to sync')
}
- const results: SyncResult[] = []
- for (const source of sources) {
- const result = await syncSingleSource(source, syncDir)
- results.push(result)
- }
+ logger.log('sync', `Syncing ${sources.length} sources...`)
- const successCount = results.filter((r) => r.success).length
- const failCount = results.filter((r) => !r.success).length
- const totalFiles = results.reduce((sum, r) => sum + (r.fileCount || 0), 0)
+ // Step 2: Sync all sources in sandbox and get snapshot
+ // This is a single step because the Sandbox object is not serializable
+ const { snapshotId, results } = await syncAllSourcesInSandbox(
+ sources,
+ config.snapshotRepo,
+ config.snapshotBranch || 'main',
+ config.githubToken,
+ )
- let pushResult = null
- if (push && successCount > 0) {
- pushResult = await pushToSnapshotStep(syncDir, config, successCount, totalFiles)
+ const successCount = results.filter(r => r.success).length
+ const failCount = results.filter(r => !r.success).length
+ const totalFiles = results.reduce((sum, r) => sum + (r.fileCount || 0), 0)
- // Trigger sandbox snapshot creation after successful push
- if (pushResult.success) {
- await triggerSnapshotStep({
- githubToken: config.githubToken, // For private repos
- snapshotRepo: config.snapshotRepo,
- snapshotBranch: config.snapshotBranch,
- })
- }
+ // Step 3: Store snapshot metadata in KV
+ const metadata: SnapshotMetadata = {
+ snapshotId,
+ createdAt: Date.now(),
+ sourceRepo: config.snapshotRepo,
}
- await cleanupWorkspace(syncDir)
+ const kv = useStorage('kv')
+ await kv.setItem(KV_KEYS.CURRENT_SNAPSHOT, metadata)
+ logger.log('sync', 'Snapshot metadata stored in KV')
// Log summary
- const logger = getLogger()
const status = failCount === 0 ? '✓' : '✗'
logger.log('sync', `${status} Done: ${successCount}/${sources.length} sources, ${totalFiles} files`)
return {
- success: failCount === 0 && (!push || pushResult?.success !== false),
+ success: failCount === 0,
+ snapshotId,
summary: {
total: sources.length,
success: successCount,
failed: failCount,
files: totalFiles,
},
- push: pushResult,
results,
}
}
diff --git a/savoir.config.ts b/savoir.config.ts
index 793b6ff..75b86f8 100644
--- a/savoir.config.ts
+++ b/savoir.config.ts
@@ -8,15 +8,13 @@ export default defineConfig({
id: 'nuxt',
label: 'Nuxt',
repo: 'nuxt/nuxt',
- additionalSyncs: [
- { repo: 'nuxt/nuxt.com', contentPath: 'content' },
- ],
+ additionalSyncs: [{ repo: 'nuxt/nuxt.com', contentPath: 'content' }],
},
{
id: 'nitro',
label: 'Nitro',
repo: 'nitrojs/nitro',
- branch: 'v3',
+ branch: 'main',
},
// Nuxt Modules