Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
some changes to build llm
Sushmithamalleshf93f0ac506c7
diff --git a/fern/scripts/build-llms-txt.ts b/fern/scripts/build-llms-txt.tsindex 0a4ba222a..6eb2d972f 100644--- a/fern/scripts/build-llms-txt.ts+++ b/fern/scripts/build-llms-txt.ts@@ -10,16 +10,16 @@ import {ensureOutputDir,type DocsConfig,type ContentItem,+ type LayoutItem,} from './lib/docs-utils';interface LlmsEntry {title: string;url: string;description?: string;- parentSection?: string; // Track parent section for nested items+ parentSection?: string; // for prefixing nested items}-// Extract title and description from MDX file frontmatterfunction extractFrontmatter(mdxPath: string): { title?: string; description?: string } {try {const fullPath = path.join(FERN_DIR, mdxPath);@@ -28,78 +28,67 @@ function extractFrontmatter(mdxPath: string): { title?: string; description?: st}const content = fs.readFileSync(fullPath, 'utf8');-- // Try to extract from frontmatterconst frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);- if (frontmatterMatch) {- const frontmatter = frontmatterMatch[1];-- // Extract title- const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);- const title = titleMatch ? titleMatch[1].trim() : undefined;-- // Extract subtitle only- const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);- const description = descMatch ? descMatch[1].trim() : undefined;-- return { title, description };++ if (!frontmatterMatch) {+ return {};}- return {};+ const frontmatter = frontmatterMatch[1];++ // Extract title from frontmatter+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);+ const title = titleMatch ? titleMatch[1].trim() : undefined;++ // Extract subtitle (used as description in llms.txt)+ const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);+ const description = descMatch ? descMatch[1].trim() : undefined;++ return { title, description };} catch (error) {return {};}}-// Process a single content item+function buildUrl(item: ContentItem, tabSlug: string, parentSlug?: string): string {+ const BASE_URL = 'https://docs.composio.dev';++ if (item.slug) {+ const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);+ return `${BASE_URL}/${urlParts.join('/')}`;+ }++ if (item.path) {+ const urlPath = item.path+ .replace(/^pages\/(dist|src)\//, '')+ .replace(/^pages\//, '')+ .replace(/\.(mdx?|md)$/, '');+ return `${BASE_URL}/${urlPath}`;+ }++ console.warn(`No slug or path for: ${item.page}`);+ return BASE_URL;+}+function processItem(item: ContentItem,tabSlug: string,parentSlug?: string,parentSection?: string): LlmsEntry | null {- // Skip items without a page name and path- if (!item.page || !item.path) {- return null;- }-- // Simple URL generation- let url: string;-- if (item.slug) {- // Use explicit slug from docs.yml- const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);- url = `https://docs.composio.dev/${urlParts.join('/')}`;- } else if (item.path) {- // Derive URL from file path- let urlPath = item.path- .replace('pages/dist/', '')- .replace('pages/src/', '')- .replace('pages/', '')- .replace('.mdx', '')- .replace('.md', '');-- url = `https://docs.composio.dev/${urlPath}`;- } else {- // This shouldn't happen - all items should have either slug or path- console.warn(`Warning: No slug or path for item: ${item.page}`);- return null;- }+ if (!item.page || !item.path) return null;- // Try to extract title and description from MDX file+ const url = buildUrl(item, tabSlug, parentSlug);const frontmatter = extractFrontmatter(item.path);- const title = frontmatter.title || item.page;- const description = frontmatter.description;-+return {- title,+ title: frontmatter.title || item.page,url,- description,+ description: frontmatter.description,parentSection,};}-// Process content items recursively (flattened)function processContentItems(items: ContentItem[],tabSlug: string,@@ -109,20 +98,23 @@ function processContentItems(const entries: LlmsEntry[] = [];for (const item of items) {- // Process the item itselfconst entry = processItem(item, tabSlug, parentSlug, parentSection);if (entry) {entries.push(entry);}- // Process nested itemsif (item.section && item.contents) {- // Combine parent slug with item slug for nested sectionsconst nestedSlug = item.slug? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug): parentSlug;- // Pass the section name down to nested items- const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);++ // pass section name to children for prefixing+ const nestedEntries = processContentItems(+ item.contents,+ tabSlug,+ nestedSlug,+ item.section+ );entries.push(...nestedEntries);}}@@ -130,9 +122,15 @@ function processContentItems(return entries;}-// Process all navigation and organize by sections+function getSectionName(tab: string, layoutItem: LayoutItem, tabDisplayName: string): string {+ if (tab === 'docs' && layoutItem.section) {+ return layoutItem.section;+ }+ return tabDisplayName;+}+function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {- const sections: Array<[string, LlmsEntry[]]> = [];+ const sections = new Map<string, LlmsEntry[]>();for (const navItem of config.navigation) {if (!navItem.layout) continue;@@ -143,122 +141,95 @@ function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {const tabSlug = skipSlug ? '' : tab;const tabDisplayName = tabConfig?.['display-name'] || tab;- // Process each layout item as a sectionfor (const layoutItem of navItem.layout) {- // Determine section name- let sectionName: string;- if (tab === 'docs' && layoutItem.section) {- sectionName = layoutItem.section;- } else if (layoutItem.section) {- sectionName = tabDisplayName; // Use tab name for non-docs sections- } else {- sectionName = tabDisplayName;- }-+ const sectionName = getSectionName(tab, layoutItem, tabDisplayName);const sectionEntries: LlmsEntry[] = [];- // Process direct pages+ // direct pageif (layoutItem.page && layoutItem.path && layoutItem.slug) {- const entry = processItem(layoutItem, tabSlug);+ const entry = processItem(layoutItem as ContentItem, tabSlug);if (entry) {sectionEntries.push(entry);}}- // Process section contents+ // nested contentsif (layoutItem.contents) {const entries = processContentItems(layoutItem.contents,tabSlug,layoutItem.slug,- undefined // Don't pass parentSection for top-level section items+ undefined // no parent section for top-level);sectionEntries.push(...entries);}if (sectionEntries.length > 0) {- // Find existing section or add new one- const existingIndex = sections.findIndex(([name]) => name === sectionName);- if (existingIndex >= 0) {- sections[existingIndex][1].push(...sectionEntries);- } else {- sections.push([sectionName, sectionEntries]);- }+ const existing = sections.get(sectionName) || [];+ sections.set(sectionName, [...existing, ...sectionEntries]);}}}- return sections;+ return Array.from(sections.entries());+}++function formatEntry(entry: LlmsEntry): string {+ const displayTitle = entry.parentSection+ ? `${entry.parentSection}: ${entry.title}`+ : entry.title;++ const link = `[${displayTitle}](${entry.url})`;+ return entry.description+ ? `- ${link}: ${entry.description}`+ : `- ${link}`;}-// Generate llms.txt content following best practicesfunction generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {let content = '# Composio\n\n';content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';- content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';+ content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n';- // Process all sectionsfor (const [sectionName, entries] of sections) {- content += `## ${sectionName}\n`;+ content += `\n## ${sectionName}\n`;- // Special handling for Toolkits section onlyif (sectionName === 'Toolkits') {- content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;+ content += '- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n';} else {for (const entry of entries) {- // Add parent section prefix if it exists- const displayTitle = entry.parentSection- ? `${entry.parentSection}: ${entry.title}`- : entry.title;-- if (entry.description) {- content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;- } else {- content += `- [${displayTitle}](${entry.url})\n`;- }+ content += formatEntry(entry) + '\n';}}- content += '\n';}return content;}-// Main functionasync function main(): Promise<void> {- console.log('π¨ Building llms.txt from docs.yml...');+ console.log('Building llms.txt...');try {- // Load docs configurationconst config = loadDocsConfig();-- // Process navigation to extract all entriesconst sections = processNavigation(config);- // Count total entries- let totalEntries = 0;- for (const [, entries] of sections) {- totalEntries += entries.length;- }-- console.log(`π Found ${totalEntries} documentation pages`);+ const totalEntries = sections.reduce((total, [_, entries]) => total + entries.length, 0);+ console.log(`Found ${totalEntries} pages across ${sections.length} sections`);- // Generate llms.txt contentconst llmsTxtContent = generateLlmsTxt(sections);-- // Write llms.txtensureOutputDir(LLMS_TXT_PATH);fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);- console.log(`β Generated llms.txt at ${LLMS_TXT_PATH}`);+ console.log(`Generated ${LLMS_TXT_PATH}`);} catch (error) {- console.error('β Error building llms.txt:', error);+ console.error('Error:', error);process.exit(1);}}-// Run the script-main().catch(error => {- console.error('β Fatal error:', error);- process.exit(1);-});\ No newline at end of file+// @ts-ignore - bun-specific+if (import.meta.main) {+ main().catch(error => {+ console.error('Fatal error:', error);+ process.exit(1);+ });+}\ No newline at end of file