Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(docs): auto-generate llms.txt from page tree instead of hardcoded lists

Sushmithamallesh9c569470c8b9
diff --git a/docs/app/llms-full.txt/route.ts b/docs/app/llms-full.txt/route.ts
index 3f8731486..3829aebd8 100644
--- a/docs/app/llms-full.txt/route.ts
+++ b/docs/app/llms-full.txt/route.ts
@@ -1,7 +1,29 @@
import { getLLMText, source, examplesSource, referenceSource, toolkitsSource } from '@/lib/source';
+import type { ReactNode } from 'react';
export const revalidate = false;
+// Fumadocs page tree node types
+interface PageNode {
+ type: 'page';
+ name: ReactNode;
+ url: string;
+}
+
+interface SeparatorNode {
+ type: 'separator';
+ name?: ReactNode;
+}
+
+interface FolderNode {
+ type: 'folder';
+ name: ReactNode;
+ index?: PageNode;
+ children: TreeNode[];
+}
+
+type TreeNode = PageNode | SeparatorNode | FolderNode;
+
// Generic page type that works for all sources
interface PageLike {
url: string;
@@ -12,13 +34,55 @@ interface PageLike {
};
}
+/**
+ * Collect page URLs from the page tree in sidebar order.
+ * This ensures pages appear in the same order as the docs sidebar.
+ */
+function collectPageUrls(nodes: TreeNode[]): string[] {
+ const urls: string[] = [];
+
+ for (const node of nodes) {
+ switch (node.type) {
+ case 'page':
+ urls.push(node.url);
+ break;
+
+ case 'folder':
+ if (node.index) {
+ urls.push(node.index.url);
+ }
+ urls.push(...collectPageUrls(node.children));
+ break;
+
+ // separators don't have URLs
+ }
+ }
+
+ return urls;
+}
+
+/**
+ * Order pages according to the page tree structure from meta.json.
+ * Pages not in the tree are appended at the end.
+ */
+function orderDocPages(pages: PageLike[], treeNodes: TreeNode[]): PageLike[] {
+ const orderedUrls = collectPageUrls(treeNodes);
+ const urlOrder = new Map(orderedUrls.map((url, i) => [url, i]));
+
+ return [...pages].sort((a, b) => {
+ const orderA = urlOrder.get(a.url) ?? 999;
+ const orderB = urlOrder.get(b.url) ?? 999;
+ return orderA - orderB;
+ });
+}
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
async function getTextForPages(pages: PageLike[]) {
return Promise.all(
pages.map(async (page) => {
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
- return await getLLMText(page as any);
+ return await getLLMText(page as any, { includeFooter: false });
} catch {
// Graceful fallback if getText fails
return `# ${page.data.title} (${page.url})\n\n${page.data.description || ''}`;
@@ -27,76 +91,12 @@ async function getTextForPages(pages: PageLike[]) {
);
}
-// Order pages according to sidebar structure from meta.json
-function orderDocPages(pages: PageLike[]) {
- // Define the order based on meta.json sidebar structure
- const sidebarOrder = [
- // Get Started
- 'index',
- 'quickstart',
- // Providers folder
- 'providers/openai-agents',
- 'providers/anthropic',
- 'providers/vercel',
- 'providers/langchain',
- 'providers/mastra',
- 'providers/openai',
- 'providers/google',
- 'providers/llamaindex',
- 'providers/crewai',
- 'providers/custom-providers',
- // Core Concepts
- 'users-and-sessions',
- 'authentication',
- 'tools-and-toolkits',
- // Getting Started
- 'configuring-sessions',
- 'authenticating-users/in-chat-authentication',
- 'authenticating-users/manually-authenticating',
- // Toolkits folder (from docs/toolkits)
- 'toolkits',
- // Guides
- 'white-labeling-authentication',
- 'managing-multiple-connected-accounts',
- 'using-custom-auth-configuration',
- // Features
- 'triggers',
- 'cli',
- 'single-toolkit-mcp',
- // Direct Tool Execution - Tools
- 'tools-direct/fetching-tools',
- 'tools-direct/authenticating-tools',
- 'tools-direct/executing-tools',
- 'tools-direct/modify-tool-behavior',
- 'tools-direct/custom-tools',
- 'tools-direct/toolkit-versioning',
- // Direct Tool Execution - Auth Configuration
- 'auth-configuration',
- // Resources
- 'debugging-info',
- 'migration-guide',
- 'troubleshooting',
- ];
-
- // Create a map for ordering
- const orderMap = new Map<string, number>();
- sidebarOrder.forEach((slug, index) => {
- orderMap.set(slug, index);
- });
-
- // Sort pages based on sidebar order, unmatched pages go to end
- return [...pages].sort((a, b) => {
- const slugA = a.slugs.join('/');
- const slugB = b.slugs.join('/');
- const orderA = orderMap.get(slugA) ?? 999;
- const orderB = orderMap.get(slugB) ?? 999;
- return orderA - orderB;
- });
-}
-
export async function GET() {
try {
- const orderedDocsPages = orderDocPages(source.getPages() as PageLike[]);
+ const orderedDocsPages = orderDocPages(
+ source.getPages() as PageLike[],
+ source.pageTree.children as TreeNode[]
+ );
const [docsResults, examplesResults, referenceResults, toolkitsResults] = await Promise.all([
getTextForPages(orderedDocsPages),
diff --git a/docs/app/llms.txt/route.ts b/docs/app/llms.txt/route.ts
index 5d59c0b8f..5b9da8450 100644
--- a/docs/app/llms.txt/route.ts
+++ b/docs/app/llms.txt/route.ts
@@ -1,105 +1,82 @@
import { source, examplesSource, referenceSource, toolkitsSource } from '@/lib/source';
+import type { ReactNode } from 'react';
export const revalidate = false;
-export async function GET() {
- try {
- const docsPages = source.getPages();
- const examplesPages = examplesSource.getPages();
- const referencePages = referenceSource.getPages();
- const toolkitsPages = toolkitsSource.getPages();
-
- // Create a map for quick lookup by slug path
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- const pageMap = new Map<string, any>();
- for (const page of docsPages) {
- pageMap.set(page.slugs.join('/'), page);
- }
-
- // Format page as simple URL
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- const formatPage = (page: any) =>
- `- https://docs.composio.dev${page.url}.md`;
-
- // Get page by slug path, return empty string if not found
- const getPage = (slugPath: string) => {
- const page = pageMap.get(slugPath);
- return page ? formatPage(page) : '';
- };
-
- // Get all pages in a folder
- const getFolderPages = (folder: string) => {
- return docsPages
- .filter(p => p.slugs[0] === folder)
- .map(formatPage)
- .join('\n');
- };
-
- // Build the index following the exact sidebar structure from meta.json
- const index = `# Composio Documentation
-
-> Composio powers 800+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
-
-## Get Started
-
-${getPage('index')}
-${getPage('quickstart')}
-
-### Providers
-
-${getFolderPages('providers')}
-
-## Core Concepts
-
-${getPage('users-and-sessions')}
-${getPage('authentication')}
-${getPage('tools-and-toolkits')}
-
-## Getting Started
-
-${getPage('configuring-sessions')}
-
-### Authenticating Users
-
-${getFolderPages('authenticating-users')}
-
-### Toolkits
-
-${getFolderPages('toolkits')}
-
-## Guides
-
-${getPage('white-labeling-authentication')}
-${getPage('managing-multiple-connected-accounts')}
-${getPage('using-custom-auth-configuration')}
-
-## Features
-
-${getPage('triggers')}
-${getPage('cli')}
-${getPage('single-toolkit-mcp')}
-
-## Direct Tool Execution Guides
+// Fumadocs page tree node types
+interface PageNode {
+ type: 'page';
+ name: ReactNode;
+ url: string;
+}
-### Tools
+interface SeparatorNode {
+ type: 'separator';
+ name?: ReactNode;
+}
-${getFolderPages('tools-direct')}
+interface FolderNode {
+ type: 'folder';
+ name: ReactNode;
+ index?: PageNode;
+ children: TreeNode[];
+}
-### Auth Configuration
+type TreeNode = PageNode | SeparatorNode | FolderNode;
+
+/**
+ * Walk the fumadocs page tree and generate a markdown index.
+ * Separators become ## headings, pages become URL entries, folders recurse.
+ */
+function walkPageTree(nodes: TreeNode[], depth = 2): string {
+ const lines: string[] = [];
+
+ for (const node of nodes) {
+ switch (node.type) {
+ case 'separator':
+ if (node.name) {
+ lines.push(`${'#'.repeat(depth)} ${String(node.name)}`, '');
+ }
+ break;
+
+ case 'page':
+ lines.push(`- https://docs.composio.dev${node.url}.md`);
+ break;
+
+ case 'folder':
+ // If folder has an index page, include it
+ if (node.index) {
+ lines.push(`- https://docs.composio.dev${node.index.url}.md`);
+ }
+ // Recurse into children
+ if (node.children.length > 0) {
+ lines.push(walkPageTree(node.children, depth + 1));
+ }
+ break;
+ }
+ }
-${getFolderPages('auth-configuration')}
+ return lines.filter(Boolean).join('\n');
+}
-## Resources
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function formatPage(page: any) {
+ return `- https://docs.composio.dev${page.url}.md`;
+}
-${getPage('debugging-info')}
+export async function GET() {
+ try {
+ const docsTree = walkPageTree(source.pageTree.children as TreeNode[]);
-### Migration Guide
+ const examplesPages = examplesSource.getPages();
+ const referencePages = referenceSource.getPages();
+ const toolkitsPages = toolkitsSource.getPages();
-${getFolderPages('migration-guide')}
+ const index = `# Composio Documentation
-### Troubleshooting
+> Composio powers 800+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
-${getFolderPages('troubleshooting')}
+${docsTree}
## Examples
diff --git a/docs/lib/source.ts b/docs/lib/source.ts
index 5544c62b1..ef6c8f2da 100644
--- a/docs/lib/source.ts
+++ b/docs/lib/source.ts
@@ -306,7 +306,8 @@ function stripTwoslashFromCodeBlocks(content: string): string {
});
}
-export async function getLLMText(page: InferPageType<typeof source>) {
+export async function getLLMText(page: InferPageType<typeof source>, options?: { includeFooter?: boolean }) {
+ const includeFooter = options?.includeFooter ?? true;
// Fall back to description if getText is not available
if (typeof page.data.getText !== 'function') {
return `# ${page.data.title} (${page.url})
@@ -337,13 +338,13 @@ ${page.data.description || ''}`;
const cleanContent = mdxToCleanMarkdown(content);
- return `# ${page.data.title} (${page.url})
-
-${cleanContent}
+ const footer = includeFooter
+ ? `\n\n---\n\n📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`
+ : '';
----
+ return `# ${page.data.title} (${page.url})
-📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`;
+${cleanContent}${footer}`;
}
export function formatDate(dateStr: string): string {