Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

update build scripts

Sushmithamalleshf639ad3ac7ca
diff --git a/fern/docs.yml b/fern/docs.yml
index 59f68855b..901ccb5a2 100644
--- a/fern/docs.yml
+++ b/fern/docs.yml
@@ -351,40 +351,54 @@ navigation:
- tab: sdk-reference
layout:
- section: Python
+ slug: python
icon: fa-brands fa-python
contents:
- page: Python SDK Reference
+ slug: python-sdk-reference
path: pages/dist/sdk/python/sdk-reference.md
- section: TypeScript
+ slug: type-script
icon: fa-brands fa-js
contents:
- section: Core Classes
slug: core-classes
contents:
- page: Composio
+ slug: composio
path: pages/dist/sdk/typescript/composio/classes/Composio.mdx
- page: ComposioProvider
+ slug: composio-provider
path: pages/dist/sdk/typescript/provider/ComposioProvider/classes/ComposioProvider.mdx
- page: OpenAIProvider
+ slug: open-ai-provider
path: pages/dist/sdk/typescript/provider/OpenAIProvider/classes/OpenAIProvider.mdx
- page: BaseAgenticProvider
+ slug: base-agentic-provider
path: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseAgenticProvider.mdx
- page: BaseNonAgenticProvider
+ slug: base-non-agentic-provider
path: pages/dist/sdk/typescript/provider/BaseProvider/classes/BaseNonAgenticProvider.mdx
- section: Models
slug: models
contents:
- page: Tools
+ slug: tools
path: pages/dist/sdk/typescript/models/Tools/classes/Tools.mdx
- page: Toolkits
+ slug: toolkits
path: pages/dist/sdk/typescript/models/Toolkits/classes/Toolkits.mdx
- page: Triggers
+ slug: triggers
path: pages/dist/sdk/typescript/models/Triggers/classes/Triggers.mdx
- page: AuthConfigs
+ slug: auth-configs
path: pages/dist/sdk/typescript/AuthConfigs/classes/AuthConfigs.mdx
- page: ConnectedAccounts
+ slug: connected-accounts
path: pages/dist/sdk/typescript/ConnectedAccounts/classes/ConnectedAccounts.mdx
- page: CustomTools
+ slug: custom-tools
path: pages/dist/sdk/typescript/CustomTools/classes/CustomTools.mdx
- tab: api-reference
layout:
diff --git a/fern/pages/src/get-started/providers.mdx b/fern/pages/src/get-started/providers.mdx
index b7fe77fb1..d50cba6e0 100644
--- a/fern/pages/src/get-started/providers.mdx
+++ b/fern/pages/src/get-started/providers.mdx
@@ -1,5 +1,6 @@
---
title: Providers
+subtitle: Use Composio with any AI framework
image: "https://og.composio.dev/api/og?title=Providers" # image for socials
keywords: ''
hide-nav-links: false
diff --git a/fern/scripts/build-llms-txt.ts b/fern/scripts/build-llms-txt.ts
index fbd726308..0a4ba222a 100644
--- a/fern/scripts/build-llms-txt.ts
+++ b/fern/scripts/build-llms-txt.ts
@@ -3,303 +3,136 @@
import fs from 'fs';
import path from 'path';
-import yaml from 'yaml';
-import { watch } from 'fs';
-
-// Define directories
-const FERN_DIR = path.join(import.meta.dir, '..');
-const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');
-const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');
-const TOOLS_JSON_PATH = path.join(
+import {
+ LLMS_TXT_PATH,
FERN_DIR,
- 'llms-txt-worker',
- 'public',
- 'robots-only',
- 'tools.json'
-);
-const PAGES_DIR = path.join(FERN_DIR, 'pages');
-const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');
-
-interface DocsConfig {
- title: string;
- navigation: Array<{
- tab: string;
- layout?: LayoutItem[];
- }>;
- tabs?: {
- [key: string]: {
- 'display-name': string;
- 'skip-slug'?: boolean;
- };
- };
-}
-
-interface LayoutItem {
- section?: string;
- contents?: ContentItem[];
- page?: string;
- path?: string;
- slug?: string;
-}
-
-interface ContentItem {
- page?: string;
- path?: string;
- slug?: string;
- section?: string;
- contents?: ContentItem[];
-}
+ loadDocsConfig,
+ ensureOutputDir,
+ type DocsConfig,
+ type ContentItem,
+} from './lib/docs-utils';
interface LlmsEntry {
title: string;
url: string;
description?: string;
+ parentSection?: string; // Track parent section for nested items
}
-interface ToolkitInfo {
- slug: string;
- description: string;
- tools: string[];
-}
-
-interface ToolsData {
- toolkitNames: string[];
- [key: string]: string[] | ToolkitInfo;
-}
-
-// Parse tool MDX files to extract toolkit information
-function parseToolMdx(toolMdxPath: string): ToolkitInfo | null {
- try {
- const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);
- if (!fs.existsSync(fullPath)) {
- console.warn(`⚠️ Tool file not found: ${fullPath}`);
- return null;
- }
-
- const content = fs.readFileSync(fullPath, 'utf8');
-
- // Extract frontmatter
- const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
- if (!frontmatterMatch) {
- console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);
- return null;
- }
-
- const frontmatter = frontmatterMatch[1];
- const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
- if (!titleMatch) {
- console.warn(`⚠️ No title found in: ${toolMdxPath}`);
- return null;
- }
-
- // Extract SLUG from content
- const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);
- if (!slugMatch) {
- console.warn(`⚠️ No SLUG found in: ${toolMdxPath}`);
- return null;
- }
-
- // Extract description
- const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);
- const description = descriptionMatch ? descriptionMatch[1].trim() : '';
-
- // Extract tool names from Accordion titles
- const toolNames: string[] = [];
- const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);
-
- for (const match of accordionMatches) {
- const toolName = match[1];
- // Only include tools that start with the toolkit prefix
- if (toolName.startsWith(slugMatch[1] + '_')) {
- toolNames.push(toolName);
- }
- }
-
- return {
- slug: slugMatch[1],
- description,
- tools: toolNames,
- };
- } catch (error) {
- console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);
- return null;
- }
-}
-
-// Extract description from MDX content
-function extractDescription(mdxPath: string): string | undefined {
+// Extract title and description from MDX file frontmatter
+function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {
try {
const fullPath = path.join(FERN_DIR, mdxPath);
if (!fs.existsSync(fullPath)) {
- console.warn(`⚠️ File not found: ${fullPath}`);
- return undefined;
+ return {};
}
const content = fs.readFileSync(fullPath, 'utf8');
- // Try to extract from frontmatter description
+ // Try to extract from frontmatter
const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
if (frontmatterMatch) {
const frontmatter = frontmatterMatch[1];
- const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);
- if (descMatch) {
- return descMatch[1].trim();
- }
+
+ // Extract title
+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
+ const title = titleMatch ? titleMatch[1].trim() : undefined;
+
+ // Extract subtitle only
+ const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
+ const description = descMatch ? descMatch[1].trim() : undefined;
+
+ return { title, description };
}
- // Extract first meaningful paragraph after the title
- const lines = content.split('\n');
- let inFrontmatter = false;
- let foundTitle = false;
- let description = '';
-
- for (const line of lines) {
- const trimmedLine = line.trim();
-
- // Handle frontmatter
- if (trimmedLine === '---') {
- inFrontmatter = !inFrontmatter;
- continue;
- }
-
- if (inFrontmatter) continue;
-
- // Skip titles
- if (trimmedLine.startsWith('#')) {
- foundTitle = true;
- continue;
- }
-
- // Skip empty lines, code blocks, and JSX
- if (
- !trimmedLine ||
- trimmedLine.startsWith('```') ||
- trimmedLine.startsWith('<') ||
- trimmedLine.startsWith('import ') ||
- trimmedLine.startsWith('export ')
- ) {
- continue;
- }
-
- // Found a paragraph
- if (foundTitle && trimmedLine.length > 20) {
- description = trimmedLine;
- break;
- }
- }
-
- // Clean up the description
- if (description) {
- // Remove MDX components and markdown syntax
- description = description
- .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags
- .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text
- .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
- .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
- .replace(/\*([^*]+)\*/g, '$1') // Remove italic
- .trim();
-
- // Truncate if too long
- if (description.length > 150) {
- description = description.substring(0, 147) + '...';
- }
-
- return description;
- }
-
- return undefined;
+ return {};
} catch (error) {
- console.warn(`⚠️ Error reading ${mdxPath}:`, error);
- return undefined;
+ return {};
}
}
-// Process content items recursively
-function processContentItems(
- items: ContentItem[],
+// Process a single content item
+function processItem(
+ item: ContentItem,
tabSlug: string,
- entries: LlmsEntry[],
- toolsData: ToolsData,
- parentSlug?: string
-): void {
- for (const item of items) {
- if (item.page && item.path) {
- // Check if this is a tool page (in toolkits/ directory) with a SLUG
- if (item.path.includes('toolkits/') && item.path.endsWith('.mdx')) {
- // Parse tool MDX file and check if it has a SLUG (actual tool page)
- const toolMdxFile = item.path.match(/toolkits\/(.+\.mdx)$/)?.[1];
- if (toolMdxFile) {
- const toolkitInfo = parseToolMdx(toolMdxFile);
- if (toolkitInfo) {
- // This is an actual tool page with SLUG - add to toolsData and skip main entries
- toolsData.toolkitNames.push(toolkitInfo.slug);
- toolsData[toolkitInfo.slug] = toolkitInfo;
- continue;
- }
- // If no SLUG found, it's a documentation page in toolkits/ - include in main entries
- }
- }
+ parentSlug?: string,
+ parentSection?: string
+): LlmsEntry | null {
+ // Skip items without a page name and path
+ if (!item.page || !item.path) {
+ return null;
+ }
- let url: string;
+ // Simple URL generation
+ let url: string;
+
+ if (item.slug) {
+ // Use explicit slug from docs.yml
+ const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
+ url = `https://docs.composio.dev/${urlParts.join('/')}`;
+ } else if (item.path) {
+ // Derive URL from file path
+ let urlPath = item.path
+ .replace('pages/dist/', '')
+ .replace('pages/src/', '')
+ .replace('pages/', '')
+ .replace('.mdx', '')
+ .replace('.md', '');
+
+ url = `https://docs.composio.dev/${urlPath}`;
+ } else {
+ // This shouldn't happen - all items should have either slug or path
+ console.warn(`Warning: No slug or path for item: ${item.page}`);
+ return null;
+ }
- if (item.slug) {
- // Use explicit slug if provided
- const slugParts = [tabSlug, item.slug].filter(Boolean);
- url = `https://docs.composio.dev/${slugParts.join('/')}`;
- // Add .mdx extension if not already present
- if (!url.endsWith('.mdx') && !url.endsWith('.md')) {
- url = url + '.mdx';
- }
- } else if (item.path) {
- // For SDK reference pages without explicit slugs, construct from path
- const pathMatch = item.path.match(/pages\/dist\/(.+)$/);
- if (pathMatch) {
- let constructedPath = pathMatch[1];
- // Handle sdk-reference special case
- if (tabSlug === 'sdk-reference' && constructedPath.startsWith('sdk/')) {
- constructedPath = constructedPath.replace('sdk/', 'sdk-reference/');
- // Convert typescript to type-script
- constructedPath = constructedPath.replace('typescript/', 'type-script/');
- }
- url = `https://docs.composio.dev/${constructedPath}`;
- } else {
- // Fallback
- const slugParts = [tabSlug, parentSlug].filter(Boolean);
- url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;
- }
- } else {
- // Fallback
- const slugParts = [tabSlug, parentSlug].filter(Boolean);
- url = `https://docs.composio.dev/${slugParts.join('/')}/${item.page.toLowerCase().replace(/\s+/g, '-')}.mdx`;
- }
+ // Try to extract title and description from MDX file
+ const frontmatter = extractFrontmatter(item.path);
+ const title = frontmatter.title || item.page;
+ const description = frontmatter.description;
- const description = extractDescription(item.path);
+ return {
+ title,
+ url,
+ description,
+ parentSection,
+ };
+}
- // Override specific page titles
- let title = item.page;
- if (title === 'Overview' && url.includes('mcp-overview')) {
- title = 'Composio MCP Servers';
- }
+// Process content items recursively (flattened)
+function processContentItems(
+ items: ContentItem[],
+ tabSlug: string,
+ parentSlug?: string,
+ parentSection?: string
+): LlmsEntry[] {
+ const entries: LlmsEntry[] = [];
- entries.push({
- title: title,
- url: url,
- description,
- });
+ for (const item of items) {
+ // Process the item itself
+ const entry = processItem(item, tabSlug, parentSlug, parentSection);
+ if (entry) {
+ entries.push(entry);
}
- // Process nested sections
+ // Process nested items
if (item.section && item.contents) {
- const newParentSlug = item.slug || parentSlug;
- processContentItems(item.contents, tabSlug, entries, toolsData, newParentSlug);
+ // Combine parent slug with item slug for nested sections
+ const nestedSlug = item.slug
+ ? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)
+ : parentSlug;
+ // Pass the section name down to nested items
+ const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);
+ entries.push(...nestedEntries);
}
}
+
+ return entries;
}
-// Process navigation items
-function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsData: ToolsData } {
- const entries: LlmsEntry[] = [];
- const toolsData: ToolsData = { toolkitNames: [] };
+// Process all navigation and organize by sections
+function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
+ const sections: Array<[string, LlmsEntry[]]> = [];
for (const navItem of config.navigation) {
if (!navItem.layout) continue;
@@ -308,144 +141,124 @@ function processNavigation(config: DocsConfig): { entries: LlmsEntry[]; toolsDat
const tabConfig = config.tabs?.[tab];
const skipSlug = tabConfig?.['skip-slug'] || false;
const tabSlug = skipSlug ? '' : tab;
+ const tabDisplayName = tabConfig?.['display-name'] || tab;
+ // Process each layout item as a section
for (const layoutItem of navItem.layout) {
+ // Determine section name
+ let sectionName: string;
+ if (tab === 'docs' && layoutItem.section) {
+ sectionName = layoutItem.section;
+ } else if (layoutItem.section) {
+ sectionName = tabDisplayName; // Use tab name for non-docs sections
+ } else {
+ sectionName = tabDisplayName;
+ }
+
+ const sectionEntries: LlmsEntry[] = [];
+
// Process direct pages
if (layoutItem.page && layoutItem.path && layoutItem.slug) {
- const slugParts = [tabSlug, layoutItem.slug].filter(Boolean);
- const url = `https://docs.composio.dev/${slugParts.join('/')}`;
- const description = extractDescription(layoutItem.path);
-
- entries.push({
- title: layoutItem.page,
- url: url.endsWith('.mdx') || url.endsWith('.md') ? url : url + '.mdx',
- description,
- });
+ const entry = processItem(layoutItem, tabSlug);
+ if (entry) {
+ sectionEntries.push(entry);
+ }
}
// Process section contents
- if (layoutItem.section && layoutItem.contents) {
- processContentItems(layoutItem.contents, tabSlug, entries, toolsData, layoutItem.slug);
+ if (layoutItem.contents) {
+ const entries = processContentItems(
+ layoutItem.contents,
+ tabSlug,
+ layoutItem.slug,
+ undefined // Don't pass parentSection for top-level section items
+ );
+ sectionEntries.push(...entries);
+ }
+
+ if (sectionEntries.length > 0) {
+ // Find existing section or add new one
+ const existingIndex = sections.findIndex(([name]) => name === sectionName);
+ if (existingIndex >= 0) {
+ sections[existingIndex][1].push(...sectionEntries);
+ } else {
+ sections.push([sectionName, sectionEntries]);
+ }
}
}
}
- return { entries, toolsData };
+ return sections;
}
-// Generate llms.txt content
-function generateLlmsTxt(entries: LlmsEntry[], toolkitCount: number): string {
- let content = '# Composio Docs\n\n## Docs\n\n';
-
- for (const entry of entries) {
- if (entry.description) {
- content += `- [${entry.title}](${entry.url}): ${entry.description}\n`;
+// Generate llms.txt content following best practices
+function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {
+ let content = '# Composio\n\n';
+ content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';
+ content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';
+
+ // Process all sections
+ for (const [sectionName, entries] of sections) {
+ content += `## ${sectionName}\n`;
+
+ // Special handling for Toolkits section only
+ if (sectionName === 'Toolkits') {
+ content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;
} else {
- content += `- [${entry.title}](${entry.url})\n`;
+ for (const entry of entries) {
+ // Add parent section prefix if it exists
+ const displayTitle = entry.parentSection
+ ? `${entry.parentSection}: ${entry.title}`
+ : entry.title;
+
+ if (entry.description) {
+ content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;
+ } else {
+ content += `- [${displayTitle}](${entry.url})\n`;
+ }
+ }
}
+ content += '\n';
}
- // Add reference to tools JSON
- content += `\n## Complete list of composio tools\n\n`;
- content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Complete JSON list of all ${toolkitCount} toolkits with their tools and descriptions\n`;
-
return content;
}
-// Main build function
-async function buildLlmsTxt(): Promise<void> {
+// Main function
+async function main(): Promise<void> {
console.log('🔨 Building llms.txt from docs.yml...');
try {
- // Read and parse docs.yml
- const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');
- const config = yaml.parse(docsYmlContent) as DocsConfig;
-
+ // Load docs configuration
+ const config = loadDocsConfig();
+
// Process navigation to extract all entries
- const { entries, toolsData } = processNavigation(config);
- console.log(`📄 Found ${entries.length} documentation pages`);
+ const sections = processNavigation(config);
+
+ // Count total entries
+ let totalEntries = 0;
+ for (const [, entries] of sections) {
+ totalEntries += entries.length;
+ }
+
+ console.log(`📄 Found ${totalEntries} documentation pages`);
// Generate llms.txt content
- const llmsTxtContent = generateLlmsTxt(entries, toolsData.toolkitNames.length);
-
- // Ensure output directory exists
- const outputDir = path.dirname(LLMS_TXT_PATH);
- if (!fs.existsSync(outputDir)) {
- fs.mkdirSync(outputDir, { recursive: true });
- }
+ const llmsTxtContent = generateLlmsTxt(sections);
// Write llms.txt
+ ensureOutputDir(LLMS_TXT_PATH);
fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);
- console.log(`✅ Generated llms.txt at ${path.relative(FERN_DIR, LLMS_TXT_PATH)}`);
-
- // Write tools.json
- const toolsDir = path.dirname(TOOLS_JSON_PATH);
- if (!fs.existsSync(toolsDir)) {
- fs.mkdirSync(toolsDir, { recursive: true });
- }
- const toolsJsonContent = JSON.stringify(toolsData, null, 2);
- fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);
- console.log(
- `✅ Generated tools.json at ${path.relative(FERN_DIR, TOOLS_JSON_PATH)} with ${toolsData.toolkitNames.length} toolkits`
- );
+
+ console.log(`✅ Generated llms.txt at ${LLMS_TXT_PATH}`);
} catch (error) {
console.error('❌ Error building llms.txt:', error);
process.exit(1);
}
}
-// Watch mode
-function startWatchMode(): void {
- console.log('👀 Watching for changes...');
-
- // Watch docs.yml
- const docsWatcher = watch(DOCS_YML_PATH, async eventType => {
- if (eventType === 'change') {
- console.log('🔄 docs.yml changed, rebuilding...');
- await buildLlmsTxt();
- }
- });
-
- // Watch MDX files in pages directory
- const pagesWatcher = watch(PAGES_DIR, { recursive: true }, async (eventType, filename) => {
- if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {
- console.log(`🔄 Page changed: ${filename}, rebuilding...`);
- await buildLlmsTxt();
- }
- });
-
- // Watch MDX files in toolkits directory
- const toolkitsWatcher = watch(TOOLKITS_DIR, { recursive: true }, async (eventType, filename) => {
- if (filename && (filename.endsWith('.mdx') || filename.endsWith('.md'))) {
- console.log(`🔄 Toolkit doc changed: ${filename}, rebuilding...`);
- await buildLlmsTxt();
- }
- });
-
- // Cleanup on exit
- process.on('SIGINT', () => {
- console.log('\n👋 Stopping watchers...');
- docsWatcher.close();
- pagesWatcher.close();
- toolkitsWatcher.close();
- process.exit(0);
- });
-}
-
-// Main function
-async function main(): Promise<void> {
- await buildLlmsTxt();
-
- // Check if we should run in watch mode
- const isWatchMode = process.argv.includes('--watch') || process.argv.includes('-w');
-
- if (isWatchMode) {
- startWatchMode();
- }
-}
-
// Run the script
main().catch(error => {
console.error('❌ Fatal error:', error);
process.exit(1);
-});
+});
\ No newline at end of file
diff --git a/fern/scripts/build-tools-json.ts b/fern/scripts/build-tools-json.ts
new file mode 100644
index 000000000..66132bf9f
--- /dev/null
+++ b/fern/scripts/build-tools-json.ts
@@ -0,0 +1,142 @@
+#!/usr/bin/env bun
+/* eslint-disable no-console */
+
+import fs from 'fs';
+import path from 'path';
+import { TOOLKITS_DIR, TOOLS_JSON_PATH, ensureOutputDir } from './lib/docs-utils';
+
+interface ToolkitInfo {
+ slug: string;
+ description: string;
+ tools: string[];
+}
+
+interface ToolsData {
+ toolkitNames: string[];
+ [key: string]: string[] | ToolkitInfo;
+}
+
+// Parse tool MDX files to extract toolkit information
+function parseToolkitMdx(toolMdxPath: string): ToolkitInfo | null {
+ try {
+ const fullPath = path.join(TOOLKITS_DIR, toolMdxPath);
+ if (!fs.existsSync(fullPath)) {
+ console.warn(`⚠️ Tool file not found: ${fullPath}`);
+ return null;
+ }
+
+ const content = fs.readFileSync(fullPath, 'utf8');
+
+ // Extract frontmatter
+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+ if (!frontmatterMatch) {
+ console.warn(`⚠️ No frontmatter found in: ${toolMdxPath}`);
+ return null;
+ }
+
+ const frontmatter = frontmatterMatch[1];
+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
+ if (!titleMatch) {
+ console.warn(`⚠️ No title found in: ${toolMdxPath}`);
+ return null;
+ }
+
+ // Extract SLUG from content
+ const slugMatch = content.match(/\*\*SLUG\*\*:\s*`([^`]+)`/);
+ if (!slugMatch) {
+ // Not a toolkit page, skip
+ return null;
+ }
+
+ // Extract description
+ const descriptionMatch = content.match(/### Description\s*\n(.+?)(?:\n|$)/);
+ const description = descriptionMatch ? descriptionMatch[1].trim() : '';
+
+ // Extract tool names from Accordion titles
+ const toolNames: string[] = [];
+ const accordionMatches = content.matchAll(/<Accordion title="([^"]+)">/g);
+
+ for (const match of accordionMatches) {
+ const toolName = match[1];
+ // Only include tools that start with the toolkit prefix
+ if (toolName.startsWith(slugMatch[1] + '_')) {
+ toolNames.push(toolName);
+ }
+ }
+
+ return {
+ slug: slugMatch[1],
+ description,
+ tools: toolNames,
+ };
+ } catch (error) {
+ console.warn(`⚠️ Error parsing tool file ${toolMdxPath}:`, error);
+ return null;
+ }
+}
+
+// Scan toolkits directory and build tools data
+function buildToolsData(): ToolsData {
+ const toolsData: ToolsData = { toolkitNames: [] };
+
+ if (!fs.existsSync(TOOLKITS_DIR)) {
+ console.warn(`⚠️ Toolkits directory not found: ${TOOLKITS_DIR}`);
+ return toolsData;
+ }
+
+ // Read all MDX files in toolkits directory
+ const files = fs.readdirSync(TOOLKITS_DIR);
+ const mdxFiles = files.filter(file => file.endsWith('.mdx'));
+
+ console.log(`📂 Found ${mdxFiles.length} MDX files in toolkits directory`);
+
+ for (const file of mdxFiles) {
+ const toolkitInfo = parseToolkitMdx(file);
+ if (toolkitInfo) {
+ toolsData.toolkitNames.push(toolkitInfo.slug);
+ toolsData[toolkitInfo.slug] = toolkitInfo;
+ console.log(` ✓ ${toolkitInfo.slug}: ${toolkitInfo.tools.length} tools`);
+ }
+ }
+
+ // Sort toolkit names alphabetically
+ toolsData.toolkitNames.sort();
+
+ return toolsData;
+}
+
+// Main function
+async function main(): Promise<void> {
+ console.log('🔨 Building tools.json...');
+
+ try {
+ // Build tools data
+ const toolsData = buildToolsData();
+
+ console.log(`\n📊 Summary:`);
+ console.log(` - Total toolkits: ${toolsData.toolkitNames.length}`);
+
+ let totalTools = 0;
+ for (const name of toolsData.toolkitNames) {
+ const toolkit = toolsData[name] as ToolkitInfo;
+ totalTools += toolkit.tools.length;
+ }
+ console.log(` - Total tools: ${totalTools}`);
+
+ // Write tools.json
+ ensureOutputDir(TOOLS_JSON_PATH);
+ const toolsJsonContent = JSON.stringify(toolsData, null, 2);
+ fs.writeFileSync(TOOLS_JSON_PATH, toolsJsonContent);
+
+ console.log(`\n✅ Generated tools.json at ${TOOLS_JSON_PATH}`);
+ } catch (error) {
+ console.error('❌ Error building tools.json:', error);
+ process.exit(1);
+ }
+}
+
+// Run the script
+main().catch(error => {
+ console.error('❌ Fatal error:', error);
+ process.exit(1);
+});
\ No newline at end of file
diff --git a/fern/scripts/lib/docs-utils.ts b/fern/scripts/lib/docs-utils.ts
new file mode 100644
index 000000000..a098c492f
--- /dev/null
+++ b/fern/scripts/lib/docs-utils.ts
@@ -0,0 +1,150 @@
+#!/usr/bin/env bun
+
+import fs from 'fs';
+import path from 'path';
+import yaml from 'yaml';
+
+// Define directories
+export const FERN_DIR = path.join(import.meta.dir, '../..');
+export const DOCS_YML_PATH = path.join(FERN_DIR, 'docs.yml');
+export const LLMS_TXT_PATH = path.join(FERN_DIR, 'llms-txt-worker', 'public', 'llms.txt');
+export const TOOLS_JSON_PATH = path.join(
+ FERN_DIR,
+ 'llms-txt-worker',
+ 'public',
+ 'robots-only',
+ 'tools.json'
+);
+export const PAGES_DIR = path.join(FERN_DIR, 'pages');
+export const TOOLKITS_DIR = path.join(FERN_DIR, 'toolkits');
+
+export interface DocsConfig {
+ title: string;
+ navigation: Array<{
+ tab: string;
+ layout?: LayoutItem[];
+ }>;
+ tabs?: {
+ [key: string]: {
+ 'display-name': string;
+ 'skip-slug'?: boolean;
+ };
+ };
+}
+
+export interface LayoutItem {
+ section?: string;
+ contents?: ContentItem[];
+ page?: string;
+ path?: string;
+ slug?: string;
+}
+
+export interface ContentItem {
+ page?: string;
+ path?: string;
+ slug?: string;
+ section?: string;
+ contents?: ContentItem[];
+}
+
+export function loadDocsConfig(): DocsConfig {
+ const docsYmlContent = fs.readFileSync(DOCS_YML_PATH, 'utf8');
+ return yaml.parse(docsYmlContent) as DocsConfig;
+}
+
+// Extract description from MDX content
+export function extractDescription(mdxPath: string): string | undefined {
+ try {
+ const fullPath = path.join(FERN_DIR, mdxPath);
+ if (!fs.existsSync(fullPath)) {
+ console.warn(`⚠️ File not found: ${fullPath}`);
+ return undefined;
+ }
+
+ const content = fs.readFileSync(fullPath, 'utf8');
+
+ // Try to extract from frontmatter description
+ const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+ if (frontmatterMatch) {
+ const frontmatter = frontmatterMatch[1];
+ const descMatch = frontmatter.match(/description:\s*["']?(.+?)["']?\s*$/m);
+ if (descMatch) {
+ return descMatch[1].trim();
+ }
+ }
+
+ // Extract first meaningful paragraph after the title
+ const lines = content.split('\n');
+ let inFrontmatter = false;
+ let foundTitle = false;
+ let description = '';
+
+ for (const line of lines) {
+ const trimmedLine = line.trim();
+
+ // Handle frontmatter
+ if (trimmedLine === '---') {
+ inFrontmatter = !inFrontmatter;
+ continue;
+ }
+
+ if (inFrontmatter) continue;
+
+ // Skip titles
+ if (trimmedLine.startsWith('#')) {
+ foundTitle = true;
+ continue;
+ }
+
+ // Skip empty lines, code blocks, and JSX
+ if (
+ !trimmedLine ||
+ trimmedLine.startsWith('```') ||
+ trimmedLine.startsWith('<') ||
+ trimmedLine.startsWith('import ') ||
+ trimmedLine.startsWith('export ')
+ ) {
+ continue;
+ }
+
+ // Found a paragraph
+ if (foundTitle && trimmedLine.length > 20) {
+ description = trimmedLine;
+ break;
+ }
+ }
+
+ // Clean up the description
+ if (description) {
+ // Remove MDX components and markdown syntax
+ description = description
+ .replace(/<[^>]+>/g, '') // Remove HTML/JSX tags
+ .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') // Convert [text](url) to text
+ .replace(/`([^`]+)`/g, '$1') // Remove inline code backticks
+ .replace(/\*\*([^*]+)\*\*/g, '$1') // Remove bold
+ .replace(/\*([^*]+)\*/g, '$1') // Remove italic
+ .trim();
+
+ // Truncate if too long
+ if (description.length > 150) {
+ description = description.substring(0, 147) + '...';
+ }
+
+ return description;
+ }
+
+ return undefined;
+ } catch (error) {
+ console.warn(`⚠️ Error reading ${mdxPath}:`, error);
+ return undefined;
+ }
+}
+
+// Ensure output directory exists
+export function ensureOutputDir(filePath: string): void {
+ const dir = path.dirname(filePath);
+ if (!fs.existsSync(dir)) {
+ fs.mkdirSync(dir, { recursive: true });
+ }
+}
\ No newline at end of file