Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
update build scripts
Sushmithamalleshf639ad3ac7ca
diff --git a/fern/docs.yml b/fern/docs.ymlindex 59f68855b..901ccb5a2 100644--- a/fern/docs.yml+++ b/fern/docs.yml@@ -351,40 +351,54 @@ navigation:- tab: sdk-referencelayout:- section: Python+ slug: pythonicon: fa-brands fa-pythoncontents:- page: Python SDK Reference+ slug: python-sdk-referencepath: pages/dist/sdk/python/sdk-reference.md- section: TypeScript+ slug: type-scripticon: fa-brands fa-jscontents:- section: Core Classesslug: core-classescontents:- page: Composio+ slug: composiopath: pages/dist/sdk/typescript/composio/classes/Composio.mdx- page: ComposioProvider+ slug: composio-providerpath: pages/dist/sdk/typescript/provider/ComposioProvider/classes/ComposioProvider.mdx- page: OpenAIProvider+ slug: open-ai-providerpath: pages/dist/sdk/typescript/provider/OpenAIProvider/classes/OpenAIProvider.mdx- page: BaseAgenticProvider+ slug: base-agentic-providerpath: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseAgenticProvider.mdx- page: BaseNonAgenticProvider+ slug: base-non-agentic-providerpath: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseNonAgenticProvider.mdx- section: Modelsslug: modelscontents:- page: Tools+ slug: toolspath: pages/dist/sdk/typescript/models/Tools/classes/Tools.mdx- page: Toolkits+ slug: toolkitspath: pages/dist/sdk/typescript/models/Toolkits/classes/Toolkits.mdx- page: Triggers+ slug: triggerspath: pages/dist/sdk/typescript/models/Triggers/classes/Triggers.mdx- page: AuthConfigs+ slug: auth-configspath: pages/dist/sdk/typescript/AuthConfigs/classes/AuthConfigs.mdx- page: ConnectedAccounts+ slug: connected-accountspath: pages/dist/sdk/typescript/ConnectedAccounts/classes/ConnectedAccounts.mdx- page: CustomTools+ slug: custom-toolspath: pages/dist/sdk/typescript/CustomTools/classes/CustomTools.mdx- tab: api-referencelayout:diff --git a/fern/pages/src/get-started/providers.mdx b/fern/pages/src/get-started/providers.mdxindex b7fe77fb1..d50cba6e0 100644--- a/fern/pages/src/get-started/providers.mdx+++ b/fern/pages/src/get-started/providers.mdx@@ -1,5 +1,6 @@---title: Providers+subtitle: Use Composio with any AI frameworkimage: "https://og.composio.dev/api/og?title=Providers" # image for socialskeywords: ''hide-nav-links: falsediff --git a/fern/scripts/build-llms-txt.ts b/fern/scripts/build-llms-txt.tsindex fbd726308..0a4ba222a 100644--- a/fern/scripts/build-llms-txt.ts+++ b/fern/scripts/build-llms-txt.ts@@ -3,303 +3,136 @@import fs from 'fs';import path from 'path';-import yaml from 'yaml';-import { watch } from 'fs';--// Define directories-const FERN_DIR = path.join(import.meta.dir, '..');-const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');-const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');-const TOOLS_JSON_PATH = path.join(+import {+ LLMS_TXT_PATH,FERN_DIR,- 'llms-txt-worker',- 'public',- 'robots-only',- 'tools.json'-);-const PAGES_DIR = path.join(FERN_DIR, 'pages');-const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');--interface DocsConfig {- title: string;- navigation: Array<{- tab: string;- layout?: LayoutItem[];- }>;- tabs?: {- [key: string]: {- 'display-name': string;- 'skip-slug'?: boolean;- };- };-}--interface LayoutItem {- section?: string;- contents?: ContentItem[];- page?: string;- path?: string;- slug?: string;-}--interface ContentItem {- page?: string;- path?: string;- slug?: string;- section?: string;- contents?: ContentItem[];-}+ loadDocsConfig,+ ensureOutputDir,+ type DocsConfig,+ type ContentItem,+} from './lib/docs-utils';interface LlmsEntry {title: string;url: string;description?: string;+ parentSection?: string; // Track parent section for nested items}-interface ToolkitInfo {- slug: string;- description: string;- tools: string[];-}--interface ToolsData {- toolkitNames: string[];- [key: string]: string[] | ToolkitInfo;-}--// Parse tool MDX files to extract toolkit information-function parseToolMdx(toolMdxPath: string): ToolkitInfo | null {- try {- const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);- if (!fs.existsSync(fullPath)) {- console.warn(`⚠️ Tool file not found: ${fullPath}`);- return null;- }-- const content = fs.readFileSync(fullPath, 'utf8');-- // Extract frontmatter- const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);- if (!frontmatterMatch) {- console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);- return null;- }-- const frontmatter = frontmatterMatch[1];- const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);- if (!titleMatch) {- console.warn(`⚠️ No title found in: ${toolMdxPath}`);- return null;- }-- // Extract SLUG from content- const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);- if (!slugMatch) {- console.warn(`⚠️ No SLUG found in: ${toolMdxPath}`);- return null;- }-- // Extract description- const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);- const description = descriptionMatch ? descriptionMatch[1].trim() : '';-- // Extract tool names from Accordion titles- const toolNames: string[] = [];- const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);-- for (const match of accordionMatches) {- const toolName = match[1];- // Only include tools that start with the toolkit prefix- if (toolName.startsWith(slugMatch[1] + '_')) {- toolNames.push(toolName);- }- }-- return {- slug: slugMatch[1],- description,- tools: toolNames,- };- } catch (error) {- console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);- return null;- }-}--// Extract description from MDX content-function extractDescription(mdxPath: string): string | undefined {+// Extract title and description from MDX file frontmatter+function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {try {const fullPath = path.join(FERN_DIR, mdxPath);if (!fs.existsSync(fullPath)) {- console.warn(`⚠️ File not found: ${fullPath}`);- return undefined;+ return {};}const content = fs.readFileSync(fullPath, 'utf8');- // Try to extract from frontmatter description+ // Try to extract from frontmatterconst frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);if (frontmatterMatch) {const frontmatter = frontmatterMatch[1];- const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);- if (descMatch) {- return descMatch[1].trim();- }++ // Extract title+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);+ const title = titleMatch ? titleMatch[1].trim() : undefined;++ // Extract subtitle only+ const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);+ const description = descMatch ? descMatch[1].trim() : undefined;++ return { title, description };}- // Extract first meaningful paragraph after the title- const lines = content.split('\n');- let inFrontmatter = false;- let foundTitle = false;- let description = '';-- for (const line of lines) {- const trimmedLine = line.trim();-- // Handle frontmatter- if (trimmedLine === '---') {- inFrontmatter = !inFrontmatter;- continue;- }-- if (inFrontmatter) continue;-- // Skip titles- if (trimmedLine.startsWith('#')) {- foundTitle = true;- continue;- }-- // Skip empty lines, code blocks, and JSX- if (- !trimmedLine ||- trimmedLine.startsWith('```') ||- trimmedLine.startsWith('<') ||- trimmedLine.startsWith('import ') ||- trimmedLine.startsWith('export ')- ) {- continue;- }-- // Found a paragraph- if (foundTitle && trimmedLine.length > 20) {- description = trimmedLine;- break;- }- }-- // Clean up the description- if (description) {- // Remove MDX components and markdown syntax- description = description- .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags- .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text- .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks- .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold- .replace(/\*([^*]+)\*/g, '$1') // Remove italic- .trim();-- // Truncate if too long- if (description.length > 150) {- description = description.substring(0, 147) + '...';- }-- return description;- }-- return undefined;+ return {};} catch (error) {- console.warn(`⚠️ Error reading ${mdxPath}:`, error);- return undefined;+ return {};}}-// Process content items recursively-function processContentItems(- items: ContentItem[],+// Process a single content item+function processItem(+ item: ContentItem,tabSlug: string,- entries: LlmsEntry[],- toolsData: ToolsData,- parentSlug?: string-): void {- for (const item of items) {- if (item.page && item.path) {- // Check if this is a tool page (in toolkits/ directory) with a SLUG- if (item.path.includes('toolkits/') && item.path.endsWith('.mdx')) {- // Parse tool MDX file and check if it has a SLUG (actual tool page)- const toolMdxFile = item.path.match(/toolkits\/(.+\.mdx)$/)?.[1];- if (toolMdxFile) {- const toolkitInfo = parseToolMdx(toolMdxFile);- if (toolkitInfo) {- // This is an actual tool page with SLUG - add to toolsData and skip main entries- toolsData.toolkitNames.push(toolkitInfo.slug);- toolsData[toolkitInfo.slug] = toolkitInfo;- continue;- }- // If no SLUG found, it's a documentation page in toolkits/ - include in main entries- }- }+ parentSlug?: string,+ parentSection?: string+): LlmsEntry | null {+ // Skip items without a page name and path+ if (!item.page || !item.path) {+ return null;+ }- let url: string;+ // Simple URL generation+ let url: string;++ if (item.slug) {+ // Use explicit slug from docs.yml+ const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);+ url = `https://docs.composio.dev/${urlParts.join('/')}`;+ } else if (item.path) {+ // Derive URL from file path+ let urlPath = item.path+ .replace('pages/dist/', '')+ .replace('pages/src/', '')+ .replace('pages/', '')+ .replace('.mdx', '')+ .replace('.md', '');++ url = `https://docs.composio.dev/${urlPath}`;+ } else {+ // This shouldn't happen - all items should have either slug or path+ console.warn(`Warning: No slug or path for item: ${item.page}`);+ return null;+ }- if (item.slug) {- // Use explicit slug if provided- const slugParts = [tabSlug, item.slug].filter(Boolean);- url = `https://docs.composio.dev/${slugParts.join('/')}`;- // Add .mdx extension if not already present- if (!url.endsWith('.mdx') && !url.endsWith('.md')) {- url = url + '.mdx';- }- } else if (item.path) {- // For SDK reference pages without explicit slugs, construct from path- const pathMatch = item.path.match(/pages\/dist\/(.+)$/);- if (pathMatch) {- let constructedPath = pathMatch[1];- // Handle sdk-reference special case- if (tabSlug === 'sdk-reference' && constructedPath.startsWith('sdk/')) {- constructedPath = constructedPath.replace('sdk/', 'sdk-reference/');- // Convert typescript to type-script- constructedPath = constructedPath.replace('typescript/', 'type-script/');- }- url = `https://docs.composio.dev/${constructedPath}`;- } else {- // Fallback- const slugParts = [tabSlug, parentSlug].filter(Boolean);- url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;- }- } else {- // Fallback- const slugParts = [tabSlug, parentSlug].filter(Boolean);- url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;- }+ // Try to extract title and description from MDX file+ const frontmatter = extractFrontmatter(item.path);+ const title = frontmatter.title || item.page;+ const description = frontmatter.description;- const description = extractDescription(item.path);+ return {+ title,+ url,+ description,+ parentSection,+ };+}- // Override specific page titles- let title = item.page;- if (title === 'Overview' && url.includes('mcp-overview')) {- title = 'Composio MCP Servers';- }+// Process content items recursively (flattened)+function processContentItems(+ items: ContentItem[],+ tabSlug: string,+ parentSlug?: string,+ parentSection?: string+): LlmsEntry[] {+ const entries: LlmsEntry[] = [];- entries.push({- title: title,- url: url,- description,- });+ for (const item of items) {+ // Process the item itself+ const entry = processItem(item, tabSlug, parentSlug, parentSection);+ if (entry) {+ entries.push(entry);}- // Process nested sections+ // Process nested itemsif (item.section && item.contents) {- const newParentSlug = item.slug || parentSlug;- processContentItems(item.contents, tabSlug, entries, toolsData, newParentSlug);+ // Combine parent slug with item slug for nested sections+ const nestedSlug = item.slug+ ? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)+ : parentSlug;+ // Pass the section name down to nested items+ const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);+ entries.push(...nestedEntries);}}++ return entries;}-// Process navigation items-function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsData: ToolsData } {- const entries: LlmsEntry[] = [];- const toolsData: ToolsData = { toolkitNames: [] };+// Process all navigation and organize by sections+function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {+ const sections: Array<[string, LlmsEntry[]]> = [];for (const navItem of config.navigation) {if (!navItem.layout) continue;@@ -308,144 +141,124 @@ function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsDatconst tabConfig = config.tabs?.[tab];const skipSlug = tabConfig?.['skip-slug'] || false;const tabSlug = skipSlug ? '' : tab;+ const tabDisplayName = tabConfig?.['display-name'] || tab;+ // Process each layout item as a sectionfor (const layoutItem of navItem.layout) {+ // Determine section name+ let sectionName: string;+ if (tab === 'docs' && layoutItem.section) {+ sectionName = layoutItem.section;+ } else if (layoutItem.section) {+ sectionName = tabDisplayName; // Use tab name for non-docs sections+ } else {+ sectionName = tabDisplayName;+ }++ const sectionEntries: LlmsEntry[] = [];+// Process direct pagesif (layoutItem.page && layoutItem.path && layoutItem.slug) {- const slugParts = [tabSlug, layoutItem.slug].filter(Boolean);- const url = `https://docs.composio.dev/${slugParts.join('/')}`;- const description = extractDescription(layoutItem.path);-- entries.push({- title: layoutItem.page,- url: url.endsWith('.mdx') || url.endsWith('.md') ? url : url + '.mdx',- description,- });+ const entry = processItem(layoutItem, tabSlug);+ if (entry) {+ sectionEntries.push(entry);+ }}// Process section contents- if (layoutItem.section && layoutItem.contents) {- processContentItems(layoutItem.contents, tabSlug, entries, toolsData, layoutItem.slug);+ if (layoutItem.contents) {+ const entries = processContentItems(+ layoutItem.contents,+ tabSlug,+ layoutItem.slug,+ undefined // Don't pass parentSection for top-level section items+ );+ sectionEntries.push(...entries);+ }++ if (sectionEntries.length > 0) {+ // Find existing section or add new one+ const existingIndex = sections.findIndex(([name]) => name === sectionName);+ if (existingIndex >= 0) {+ sections[existingIndex][1].push(...sectionEntries);+ } else {+ sections.push([sectionName, sectionEntries]);+ }}}}- return { entries, toolsData };+ return sections;}-// Generate llms.txt content-function generateLlmsTxt(entries: LlmsEntry[], toolkitCount: number): string {- let content = '# Composio Docs\n\n## Docs\n\n';-- for (const entry of entries) {- if (entry.description) {- content += `- [${entry.title}](${entry.url}): ${entry.description}\n`;+// Generate llms.txt content following best practices+function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {+ let content = '# Composio\n\n';+ content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';+ content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';++ // Process all sections+ for (const [sectionName, entries] of sections) {+ content += `## ${sectionName}\n`;++ // Special handling for Toolkits section only+ if (sectionName === 'Toolkits') {+ content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;} else {- content += `- [${entry.title}](${entry.url})\n`;+ for (const entry of entries) {+ // Add parent section prefix if it exists+ const displayTitle = entry.parentSection+ ? `${entry.parentSection}: ${entry.title}`+ : entry.title;++ if (entry.description) {+ content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;+ } else {+ content += `- [${displayTitle}](${entry.url})\n`;+ }+ }}+ content += '\n';}- // Add reference to tools JSON- content += `\n## Complete list of composio tools\n\n`;- content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Complete JSON list of all ${toolkitCount} toolkits with their tools and descriptions\n`;-return content;}-// Main build function-async function buildLlmsTxt(): Promise<void> {+// Main function+async function main(): Promise<void> {console.log('🔨 Building llms.txt from docs.yml...');try {- // Read and parse docs.yml- const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');- const config = yaml.parse(docsYmlContent) as DocsConfig;-+ // Load docs configuration+ const config = loadDocsConfig();+// Process navigation to extract all entries- const { entries, toolsData } = processNavigation(config);- console.log(`📄 Found ${entries.length} documentation pages`);+ const sections = processNavigation(config);++ // Count total entries+ let totalEntries = 0;+ for (const [, entries] of sections) {+ totalEntries += entries.length;+ }++ console.log(`📄 Found ${totalEntries} documentation pages`);// Generate llms.txt content- const llmsTxtContent = generateLlmsTxt(entries, toolsData.toolkitNames.length);-- // Ensure output directory exists- const outputDir = path.dirname(LLMS_TXT_PATH);- if (!fs.existsSync(outputDir)) {- fs.mkdirSync(outputDir, { recursive: true });- }+ const llmsTxtContent = generateLlmsTxt(sections);// Write llms.txt+ ensureOutputDir(LLMS_TXT_PATH);fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);- console.log(`✅ Generated llms.txt at ${path.relative(FERN_DIR, LLMS_TXT_PATH)}`);-- // Write tools.json- const toolsDir = path.dirname(TOOLS_JSON_PATH);- if (!fs.existsSync(toolsDir)) {- fs.mkdirSync(toolsDir, { recursive: true });- }- const toolsJsonContent = JSON.stringify(toolsData, null, 2);- fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);- console.log(- `✅ Generated tools.json at ${path.relative(FERN_DIR, TOOLS_JSON_PATH)} with ${toolsData.toolkitNames.length} toolkits`- );++ console.log(`✅ Generated llms.txt at ${LLMS_TXT_PATH}`);} catch (error) {console.error('❌ Error building llms.txt:', error);process.exit(1);}}-// Watch mode-function startWatchMode(): void {- console.log('👀 Watching for changes...');-- // Watch docs.yml- const docsWatcher = watch(DOCS_YML_PATH, async eventType => {- if (eventType === 'change') {- console.log('🔄 docs.yml changed, rebuilding...');- await buildLlmsTxt();- }- });-- // Watch MDX files in pages directory- const pagesWatcher = watch(PAGES_DIR, { recursive: true }, async (eventType, filename) => {- if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {- console.log(`🔄 Page changed: ${filename}, rebuilding...`);- await buildLlmsTxt();- }- });-- // Watch MDX files in toolkits directory- const toolkitsWatcher = watch(TOOLKITS_DIR, { recursive: true }, async (eventType, filename) => {- if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {- console.log(`🔄 Toolkit doc changed: ${filename}, rebuilding...`);- await buildLlmsTxt();- }- });-- // Cleanup on exit- process.on('SIGINT', () => {- console.log('\n👋 Stopping watchers...');- docsWatcher.close();- pagesWatcher.close();- toolkitsWatcher.close();- process.exit(0);- });-}--// Main function-async function main(): Promise<void> {- await buildLlmsTxt();-- // Check if we should run in watch mode- const isWatchMode = process.argv.includes('--watch') || process.argv.includes('-w');-- if (isWatchMode) {- startWatchMode();- }-}-// Run the scriptmain().catch(error => {console.error('❌ Fatal error:', error);process.exit(1);-});+});\ No newline at end of filediff --git a/fern/scripts/build-tools-json.ts b/fern/scripts/build-tools-json.tsnew file mode 100644index 000000000..66132bf9f--- /dev/null+++ b/fern/scripts/build-tools-json.ts@@ -0,0 +1,142 @@+#!/usr/bin/env bun+/* eslint-disable no-console */++import fs from 'fs';+import path from 'path';+import { TOOLKITS_DIR, TOOLS_JSON_PATH, ensureOutputDir } from './lib/docs-utils';++interface ToolkitInfo {+ slug: string;+ description: string;+ tools: string[];+}++interface ToolsData {+ toolkitNames: string[];+ [key: string]: string[] | ToolkitInfo;+}++// Parse tool MDX files to extract toolkit information+function parseToolkitMdx(toolMdxPath: string): ToolkitInfo | null {+ try {+ const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);+ if (!fs.existsSync(fullPath)) {+ console.warn(`⚠️ Tool file not found: ${fullPath}`);+ return null;+ }++ const content = fs.readFileSync(fullPath, 'utf8');++ // Extract frontmatter+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);+ if (!frontmatterMatch) {+ console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);+ return null;+ }++ const frontmatter = frontmatterMatch[1];+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);+ if (!titleMatch) {+ console.warn(`⚠️ No title found in: ${toolMdxPath}`);+ return null;+ }++ // Extract SLUG from content+ const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);+ if (!slugMatch) {+ // Not a toolkit page, skip+ return null;+ }++ // Extract description+ const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);+ const description = descriptionMatch ? descriptionMatch[1].trim() : '';++ // Extract tool names from Accordion titles+ const toolNames: string[] = [];+ const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);++ for (const match of accordionMatches) {+ const toolName = match[1];+ // Only include tools that start with the toolkit prefix+ if (toolName.startsWith(slugMatch[1] + '_')) {+ toolNames.push(toolName);+ }+ }++ return {+ slug: slugMatch[1],+ description,+ tools: toolNames,+ };+ } catch (error) {+ console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);+ return null;+ }+}++// Scan toolkits directory and build tools data+function buildToolsData(): ToolsData {+ const toolsData: ToolsData = { toolkitNames: [] };++ if (!fs.existsSync(TOOLKITS_DIR)) {+ console.warn(`⚠️ Toolkits directory not found: ${TOOLKITS_DIR}`);+ return toolsData;+ }++ // Read all MDX files in toolkits directory+ const files = fs.readdirSync(TOOLKITS_DIR);+ const mdxFiles = files.filter(file => file.endsWith('.mdx'));++ console.log(`📂 Found ${mdxFiles.length} MDX files in toolkits directory`);++ for (const file of mdxFiles) {+ const toolkitInfo = parseToolkitMdx(file);+ if (toolkitInfo) {+ toolsData.toolkitNames.push(toolkitInfo.slug);+ toolsData[toolkitInfo.slug] = toolkitInfo;+ console.log(` ✓ ${toolkitInfo.slug}: ${toolkitInfo.tools.length} tools`);+ }+ }++ // Sort toolkit names alphabetically+ toolsData.toolkitNames.sort();++ return toolsData;+}++// Main function+async function main(): Promise<void> {+ console.log('🔨 Building tools.json...');++ try {+ // Build tools data+ const toolsData = buildToolsData();++ console.log(`\n📊 Summary:`);+ console.log(` - Total toolkits: ${toolsData.toolkitNames.length}`);++ let totalTools = 0;+ for (const name of toolsData.toolkitNames) {+ const toolkit = toolsData[name] as ToolkitInfo;+ totalTools += toolkit.tools.length;+ }+ console.log(` - Total tools: ${totalTools}`);++ // Write tools.json+ ensureOutputDir(TOOLS_JSON_PATH);+ const toolsJsonContent = JSON.stringify(toolsData, null, 2);+ fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);++ console.log(`\n✅ Generated tools.json at ${TOOLS_JSON_PATH}`);+ } catch (error) {+ console.error('❌ Error building tools.json:', error);+ process.exit(1);+ }+}++// Run the script+main().catch(error => {+ console.error('❌ Fatal error:', error);+ process.exit(1);+});\ No newline at end of filediff --git a/fern/scripts/lib/docs-utils.ts b/fern/scripts/lib/docs-utils.tsnew file mode 100644index 000000000..a098c492f--- /dev/null+++ b/fern/scripts/lib/docs-utils.ts@@ -0,0 +1,150 @@+#!/usr/bin/env bun++import fs from 'fs';+import path from 'path';+import yaml from 'yaml';++// Define directories+export const FERN_DIR = path.join(import.meta.dir, '../..');+export const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');+export const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');+export const TOOLS_JSON_PATH = path.join(+ FERN_DIR,+ 'llms-txt-worker',+ 'public',+ 'robots-only',+ 'tools.json'+);+export const PAGES_DIR = path.join(FERN_DIR, 'pages');+export const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');++export interface DocsConfig {+ title: string;+ navigation: Array<{+ tab: string;+ layout?: LayoutItem[];+ }>;+ tabs?: {+ [key: string]: {+ 'display-name': string;+ 'skip-slug'?: boolean;+ };+ };+}++export interface LayoutItem {+ section?: string;+ contents?: ContentItem[];+ page?: string;+ path?: string;+ slug?: string;+}++export interface ContentItem {+ page?: string;+ path?: string;+ slug?: string;+ section?: string;+ contents?: ContentItem[];+}++export function loadDocsConfig(): DocsConfig {+ const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');+ return yaml.parse(docsYmlContent) as DocsConfig;+}++// Extract description from MDX content+export function extractDescription(mdxPath: string): string | undefined {+ try {+ const fullPath = path.join(FERN_DIR, mdxPath);+ if (!fs.existsSync(fullPath)) {+ console.warn(`⚠️ File not found: ${fullPath}`);+ return undefined;+ }++ const content = fs.readFileSync(fullPath, 'utf8');++ // Try to extract from frontmatter description+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);+ if (frontmatterMatch) {+ const frontmatter = frontmatterMatch[1];+ const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);+ if (descMatch) {+ return descMatch[1].trim();+ }+ }++ // Extract first meaningful paragraph after the title+ const lines = content.split('\n');+ let inFrontmatter = false;+ let foundTitle = false;+ let description = '';++ for (const line of lines) {+ const trimmedLine = line.trim();++ // Handle frontmatter+ if (trimmedLine === '---') {+ inFrontmatter = !inFrontmatter;+ continue;+ }++ if (inFrontmatter) continue;++ // Skip titles+ if (trimmedLine.startsWith('#')) {+ foundTitle = true;+ continue;+ }++ // Skip empty lines, code blocks, and JSX+ if (+ !trimmedLine ||+ trimmedLine.startsWith('```') ||+ trimmedLine.startsWith('<') ||+ trimmedLine.startsWith('import ') ||+ trimmedLine.startsWith('export ')+ ) {+ continue;+ }++ // Found a paragraph+ if (foundTitle && trimmedLine.length > 20) {+ description = trimmedLine;+ break;+ }+ }++ // Clean up the description+ if (description) {+ // Remove MDX components and markdown syntax+ description = description+ .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags+ .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text+ .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks+ .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold+ .replace(/\*([^*]+)\*/g, '$1') // Remove italic+ .trim();++ // Truncate if too long+ if (description.length > 150) {+ description = description.substring(0, 147) + '...';+ }++ return description;+ }++ return undefined;+ } catch (error) {+ console.warn(`⚠️ Error reading ${mdxPath}:`, error);+ return undefined;+ }+}++// Ensure output directory exists+export function ensureOutputDir(filePath: string): void {+ const dir = path.dirname(filePath);+ if (!fs.existsSync(dir)) {+ fs.mkdirSync(dir, { recursive: true });+ }+}\ No newline at end of file