Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

some changes to build llm

Sushmithamalleshf93f0ac506c7
diff --git a/fern/scripts/build-llms-txt.ts b/fern/scripts/build-llms-txt.ts
index 0a4ba222a..6eb2d972f 100644
--- a/fern/scripts/build-llms-txt.ts
+++ b/fern/scripts/build-llms-txt.ts
@@ -10,16 +10,16 @@ import {
ensureOutputDir,
type DocsConfig,
type ContentItem,
+ type LayoutItem,
} from './lib/docs-utils';
interface LlmsEntry {
title: string;
url: string;
description?: string;
- parentSection?: string; // Track parent section for nested items
+ parentSection?: string; // for prefixing nested items
}
-// Extract title and description from MDX file frontmatter
function extractFrontmatter(mdxPath: string): { title?: string; description?: string } {
try {
const fullPath = path.join(FERN_DIR, mdxPath);
@@ -28,78 +28,67 @@ function extractFrontmatter(mdxPath: string): { title?: string; description?: st
}
const content = fs.readFileSync(fullPath, 'utf8');
-
- // Try to extract from frontmatter
const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
- if (frontmatterMatch) {
- const frontmatter = frontmatterMatch[1];
-
- // Extract title
- const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
- const title = titleMatch ? titleMatch[1].trim() : undefined;
-
- // Extract subtitle only
- const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
- const description = descMatch ? descMatch[1].trim() : undefined;
-
- return { title, description };
+
+ if (!frontmatterMatch) {
+ return {};
}
- return {};
+ const frontmatter = frontmatterMatch[1];
+
+ // Extract title from frontmatter
+ const titleMatch = frontmatter.match(/title:\s*["']?(.+?)["']?\s*$/m);
+ const title = titleMatch ? titleMatch[1].trim() : undefined;
+
+ // Extract subtitle (used as description in llms.txt)
+ const descMatch = frontmatter.match(/subtitle:\s*["']?(.+?)["']?\s*$/m);
+ const description = descMatch ? descMatch[1].trim() : undefined;
+
+ return { title, description };
} catch (error) {
return {};
}
}
-// Process a single content item
+function buildUrl(item: ContentItem, tabSlug: string, parentSlug?: string): string {
+ const BASE_URL = 'https://docs.composio.dev';
+
+ if (item.slug) {
+ const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
+ return `${BASE_URL}/${urlParts.join('/')}`;
+ }
+
+ if (item.path) {
+ const urlPath = item.path
+ .replace(/^pages\/(dist|src)\//, '')
+ .replace(/^pages\//, '')
+ .replace(/\.(mdx?|md)$/, '');
+ return `${BASE_URL}/${urlPath}`;
+ }
+
+ console.warn(`No slug or path for: ${item.page}`);
+ return BASE_URL;
+}
+
function processItem(
item: ContentItem,
tabSlug: string,
parentSlug?: string,
parentSection?: string
): LlmsEntry | null {
- // Skip items without a page name and path
- if (!item.page || !item.path) {
- return null;
- }
-
- // Simple URL generation
- let url: string;
-
- if (item.slug) {
- // Use explicit slug from docs.yml
- const urlParts = [tabSlug, parentSlug, item.slug].filter(Boolean);
- url = `https://docs.composio.dev/${urlParts.join('/')}`;
- } else if (item.path) {
- // Derive URL from file path
- let urlPath = item.path
- .replace('pages/dist/', '')
- .replace('pages/src/', '')
- .replace('pages/', '')
- .replace('.mdx', '')
- .replace('.md', '');
-
- url = `https://docs.composio.dev/${urlPath}`;
- } else {
- // This shouldn't happen - all items should have either slug or path
- console.warn(`Warning: No slug or path for item: ${item.page}`);
- return null;
- }
+ if (!item.page || !item.path) return null;
- // Try to extract title and description from MDX file
+ const url = buildUrl(item, tabSlug, parentSlug);
const frontmatter = extractFrontmatter(item.path);
- const title = frontmatter.title || item.page;
- const description = frontmatter.description;
-
+
return {
- title,
+ title: frontmatter.title || item.page,
url,
- description,
+ description: frontmatter.description,
parentSection,
};
}
-// Process content items recursively (flattened)
function processContentItems(
items: ContentItem[],
tabSlug: string,
@@ -109,20 +98,23 @@ function processContentItems(
const entries: LlmsEntry[] = [];
for (const item of items) {
- // Process the item itself
const entry = processItem(item, tabSlug, parentSlug, parentSection);
if (entry) {
entries.push(entry);
}
- // Process nested items
if (item.section && item.contents) {
- // Combine parent slug with item slug for nested sections
const nestedSlug = item.slug
? (parentSlug ? `${parentSlug}/${item.slug}` : item.slug)
: parentSlug;
- // Pass the section name down to nested items
- const nestedEntries = processContentItems(item.contents, tabSlug, nestedSlug, item.section);
+
+ // pass section name to children for prefixing
+ const nestedEntries = processContentItems(
+ item.contents,
+ tabSlug,
+ nestedSlug,
+ item.section
+ );
entries.push(...nestedEntries);
}
}
@@ -130,9 +122,15 @@ function processContentItems(
return entries;
}
-// Process all navigation and organize by sections
+function getSectionName(tab: string, layoutItem: LayoutItem, tabDisplayName: string): string {
+ if (tab === 'docs' && layoutItem.section) {
+ return layoutItem.section;
+ }
+ return tabDisplayName;
+}
+
function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
- const sections: Array<[string, LlmsEntry[]]> = [];
+ const sections = new Map<string, LlmsEntry[]>();
for (const navItem of config.navigation) {
if (!navItem.layout) continue;
@@ -143,122 +141,95 @@ function processNavigation(config: DocsConfig): Array<[string, LlmsEntry[]]> {
const tabSlug = skipSlug ? '' : tab;
const tabDisplayName = tabConfig?.['display-name'] || tab;
- // Process each layout item as a section
for (const layoutItem of navItem.layout) {
- // Determine section name
- let sectionName: string;
- if (tab === 'docs' && layoutItem.section) {
- sectionName = layoutItem.section;
- } else if (layoutItem.section) {
- sectionName = tabDisplayName; // Use tab name for non-docs sections
- } else {
- sectionName = tabDisplayName;
- }
-
+ const sectionName = getSectionName(tab, layoutItem, tabDisplayName);
const sectionEntries: LlmsEntry[] = [];
- // Process direct pages
+ // direct page
if (layoutItem.page && layoutItem.path && layoutItem.slug) {
- const entry = processItem(layoutItem, tabSlug);
+ const entry = processItem(layoutItem as ContentItem, tabSlug);
if (entry) {
sectionEntries.push(entry);
}
}
- // Process section contents
+ // nested contents
if (layoutItem.contents) {
const entries = processContentItems(
layoutItem.contents,
tabSlug,
layoutItem.slug,
- undefined // Don't pass parentSection for top-level section items
+ undefined // no parent section for top-level
);
sectionEntries.push(...entries);
}
if (sectionEntries.length > 0) {
- // Find existing section or add new one
- const existingIndex = sections.findIndex(([name]) => name === sectionName);
- if (existingIndex >= 0) {
- sections[existingIndex][1].push(...sectionEntries);
- } else {
- sections.push([sectionName, sectionEntries]);
- }
+ const existing = sections.get(sectionName) || [];
+ sections.set(sectionName, [...existing, ...sectionEntries]);
}
}
}
- return sections;
+ return Array.from(sections.entries());
+}
+
+function formatEntry(entry: LlmsEntry): string {
+ const displayTitle = entry.parentSection
+ ? `${entry.parentSection}: ${entry.title}`
+ : entry.title;
+
+ const link = `[${displayTitle}](${entry.url})`;
+ return entry.description
+ ? `- ${link}: ${entry.description}`
+ : `- ${link}`;
}
-// Generate llms.txt content following best practices
function generateLlmsTxt(sections: Array<[string, LlmsEntry[]]>): string {
let content = '# Composio\n\n';
content += '> Composio is a platform that connects AI agents to 500+ external tools and services through managed authentication, tool execution, and triggers. It simplifies integrating AI agents with SaaS applications, developer tools, and custom APIs.\n\n';
- content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n\n';
+ content += 'Composio provides managed OAuth and API key authentication, tool execution across multiple AI frameworks (OpenAI, Anthropic, LangChain, CrewAI, etc.), MCP server support, and webhooks/triggers for real-time agent actions.\n';
- // Process all sections
for (const [sectionName, entries] of sections) {
- content += `## ${sectionName}\n`;
+ content += `\n## ${sectionName}\n`;
- // Special handling for Toolkits section only
if (sectionName === 'Toolkits') {
- content += `- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n`;
+ content += '- [Tools JSON](https://docs.composio.dev/robots-only/tools.json): Machine-readable JSON with all 700+ toolkit slugs, descriptions, and tool names\n';
} else {
for (const entry of entries) {
- // Add parent section prefix if it exists
- const displayTitle = entry.parentSection
- ? `${entry.parentSection}: ${entry.title}`
- : entry.title;
-
- if (entry.description) {
- content += `- [${displayTitle}](${entry.url}): ${entry.description}\n`;
- } else {
- content += `- [${displayTitle}](${entry.url})\n`;
- }
+ content += formatEntry(entry) + '\n';
}
}
- content += '\n';
}
return content;
}
-// Main function
async function main(): Promise<void> {
- console.log('πŸ”¨ Building llms.txt from docs.yml...');
+ console.log('Building llms.txt...');
try {
- // Load docs configuration
const config = loadDocsConfig();
-
- // Process navigation to extract all entries
const sections = processNavigation(config);
- // Count total entries
- let totalEntries = 0;
- for (const [, entries] of sections) {
- totalEntries += entries.length;
- }
-
- console.log(`πŸ“„ Found ${totalEntries} documentation pages`);
+ const totalEntries = sections.reduce((total, [_, entries]) => total + entries.length, 0);
+ console.log(`Found ${totalEntries} pages across ${sections.length} sections`);
- // Generate llms.txt content
const llmsTxtContent = generateLlmsTxt(sections);
-
- // Write llms.txt
ensureOutputDir(LLMS_TXT_PATH);
fs.writeFileSync(LLMS_TXT_PATH, llmsTxtContent);
- console.log(`βœ… Generated llms.txt at ${LLMS_TXT_PATH}`);
+ console.log(`Generated ${LLMS_TXT_PATH}`);
} catch (error) {
- console.error('❌ Error building llms.txt:', error);
+ console.error('Error:', error);
process.exit(1);
}
}
-// Run the script
-main().catch(error => {
- console.error('❌ Fatal error:', error);
- process.exit(1);
-});
\ No newline at end of file
+// @ts-ignore - bun-specific
+if (import.meta.main) {
+ main().catch(error => {
+ console.error('Fatal error:', error);
+ process.exit(1);
+ });
+}
\ No newline at end of file