Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(docs): auto-generate llms.txt from page tree instead of hardcoded lists
Sushmithamallesh9c569470c8b9
diff --git a/docs/app/llms-full.txt/route.ts b/docs/app/llms-full.txt/route.tsindex 3f8731486..3829aebd8 100644--- a/docs/app/llms-full.txt/route.ts+++ b/docs/app/llms-full.txt/route.ts@@ -1,7 +1,29 @@import { getLLMText, source, examplesSource, referenceSource, toolkitsSource } from '@/lib/source';+import type { ReactNode } from 'react';export const revalidate = false;+// Fumadocs page tree node types+interface PageNode {+ type: 'page';+ name: ReactNode;+ url: string;+}++interface SeparatorNode {+ type: 'separator';+ name?: ReactNode;+}++interface FolderNode {+ type: 'folder';+ name: ReactNode;+ index?: PageNode;+ children: TreeNode[];+}++type TreeNode = PageNode | SeparatorNode | FolderNode;+// Generic page type that works for all sourcesinterface PageLike {url: string;@@ -12,13 +34,55 @@ interface PageLike {};}+/**+ * Collect page URLs from the page tree in sidebar order.+ * This ensures pages appear in the same order as the docs sidebar.+ */+function collectPageUrls(nodes: TreeNode[]): string[] {+ const urls: string[] = [];++ for (const node of nodes) {+ switch (node.type) {+ case 'page':+ urls.push(node.url);+ break;++ case 'folder':+ if (node.index) {+ urls.push(node.index.url);+ }+ urls.push(...collectPageUrls(node.children));+ break;++ // separators don't have URLs+ }+ }++ return urls;+}++/**+ * Order pages according to the page tree structure from meta.json.+ * Pages not in the tree are appended at the end.+ */+function orderDocPages(pages: PageLike[], treeNodes: TreeNode[]): PageLike[] {+ const orderedUrls = collectPageUrls(treeNodes);+ const urlOrder = new Map(orderedUrls.map((url, i) => [url, i]));++ return [...pages].sort((a, b) => {+ const orderA = urlOrder.get(a.url) ?? 999;+ const orderB = urlOrder.get(b.url) ?? 999;+ return orderA - orderB;+ });+}+// eslint-disable-next-line @typescript-eslint/no-explicit-anyasync function getTextForPages(pages: PageLike[]) {return Promise.all(pages.map(async (page) => {try {// eslint-disable-next-line @typescript-eslint/no-explicit-any- return await getLLMText(page as any);+ return await getLLMText(page as any, { includeFooter: false });} catch {// Graceful fallback if getText failsreturn `# ${page.data.title} (${page.url})\n\n${page.data.description || ''}`;@@ -27,76 +91,12 @@ async function getTextForPages(pages: PageLike[]) {);}-// Order pages according to sidebar structure from meta.json-function orderDocPages(pages: PageLike[]) {- // Define the order based on meta.json sidebar structure- const sidebarOrder = [- // Get Started- 'index',- 'quickstart',- // Providers folder- 'providers/openai-agents',- 'providers/anthropic',- 'providers/vercel',- 'providers/langchain',- 'providers/mastra',- 'providers/openai',- 'providers/google',- 'providers/llamaindex',- 'providers/crewai',- 'providers/custom-providers',- // Core Concepts- 'users-and-sessions',- 'authentication',- 'tools-and-toolkits',- // Getting Started- 'configuring-sessions',- 'authenticating-users/in-chat-authentication',- 'authenticating-users/manually-authenticating',- // Toolkits folder (from docs/toolkits)- 'toolkits',- // Guides- 'white-labeling-authentication',- 'managing-multiple-connected-accounts',- 'using-custom-auth-configuration',- // Features- 'triggers',- 'cli',- 'single-toolkit-mcp',- // Direct Tool Execution - Tools- 'tools-direct/fetching-tools',- 'tools-direct/authenticating-tools',- 'tools-direct/executing-tools',- 'tools-direct/modify-tool-behavior',- 'tools-direct/custom-tools',- 'tools-direct/toolkit-versioning',- // Direct Tool Execution - Auth Configuration- 'auth-configuration',- // Resources- 'debugging-info',- 'migration-guide',- 'troubleshooting',- ];-- // Create a map for ordering- const orderMap = new Map<string, number>();- sidebarOrder.forEach((slug, index) => {- orderMap.set(slug, index);- });-- // Sort pages based on sidebar order, unmatched pages go to end- return [...pages].sort((a, b) => {- const slugA = a.slugs.join('/');- const slugB = b.slugs.join('/');- const orderA = orderMap.get(slugA) ?? 999;- const orderB = orderMap.get(slugB) ?? 999;- return orderA - orderB;- });-}-export async function GET() {try {- const orderedDocsPages = orderDocPages(source.getPages() as PageLike[]);+ const orderedDocsPages = orderDocPages(+ source.getPages() as PageLike[],+ source.pageTree.children as TreeNode[]+ );const [docsResults, examplesResults, referenceResults, toolkitsResults] = await Promise.all([getTextForPages(orderedDocsPages),diff --git a/docs/app/llms.txt/route.ts b/docs/app/llms.txt/route.tsindex 5d59c0b8f..5b9da8450 100644--- a/docs/app/llms.txt/route.ts+++ b/docs/app/llms.txt/route.ts@@ -1,105 +1,82 @@import { source, examplesSource, referenceSource, toolkitsSource } from '@/lib/source';+import type { ReactNode } from 'react';export const revalidate = false;-export async function GET() {- try {- const docsPages = source.getPages();- const examplesPages = examplesSource.getPages();- const referencePages = referenceSource.getPages();- const toolkitsPages = toolkitsSource.getPages();-- // Create a map for quick lookup by slug path- // eslint-disable-next-line @typescript-eslint/no-explicit-any- const pageMap = new Map<string, any>();- for (const page of docsPages) {- pageMap.set(page.slugs.join('/'), page);- }-- // Format page as simple URL- // eslint-disable-next-line @typescript-eslint/no-explicit-any- const formatPage = (page: any) =>- `- https://docs.composio.dev${page.url}.md`;-- // Get page by slug path, return empty string if not found- const getPage = (slugPath: string) => {- const page = pageMap.get(slugPath);- return page ? formatPage(page) : '';- };-- // Get all pages in a folder- const getFolderPages = (folder: string) => {- return docsPages- .filter(p => p.slugs[0] === folder)- .map(formatPage)- .join('\n');- };-- // Build the index following the exact sidebar structure from meta.json- const index = `# Composio Documentation--> Composio powers 800+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.--## Get Started--${getPage('index')}-${getPage('quickstart')}--### Providers--${getFolderPages('providers')}--## Core Concepts--${getPage('users-and-sessions')}-${getPage('authentication')}-${getPage('tools-and-toolkits')}--## Getting Started--${getPage('configuring-sessions')}--### Authenticating Users--${getFolderPages('authenticating-users')}--### Toolkits--${getFolderPages('toolkits')}--## Guides--${getPage('white-labeling-authentication')}-${getPage('managing-multiple-connected-accounts')}-${getPage('using-custom-auth-configuration')}--## Features--${getPage('triggers')}-${getPage('cli')}-${getPage('single-toolkit-mcp')}--## Direct Tool Execution Guides+// Fumadocs page tree node types+interface PageNode {+ type: 'page';+ name: ReactNode;+ url: string;+}-### Tools+interface SeparatorNode {+ type: 'separator';+ name?: ReactNode;+}-${getFolderPages('tools-direct')}+interface FolderNode {+ type: 'folder';+ name: ReactNode;+ index?: PageNode;+ children: TreeNode[];+}-### Auth Configuration+type TreeNode = PageNode | SeparatorNode | FolderNode;++/**+ * Walk the fumadocs page tree and generate a markdown index.+ * Separators become ## headings, pages become URL entries, folders recurse.+ */+function walkPageTree(nodes: TreeNode[], depth = 2): string {+ const lines: string[] = [];++ for (const node of nodes) {+ switch (node.type) {+ case 'separator':+ if (node.name) {+ lines.push(`${'#'.repeat(depth)} ${String(node.name)}`, '');+ }+ break;++ case 'page':+ lines.push(`- https://docs.composio.dev${node.url}.md`);+ break;++ case 'folder':+ // If folder has an index page, include it+ if (node.index) {+ lines.push(`- https://docs.composio.dev${node.index.url}.md`);+ }+ // Recurse into children+ if (node.children.length > 0) {+ lines.push(walkPageTree(node.children, depth + 1));+ }+ break;+ }+ }-${getFolderPages('auth-configuration')}+ return lines.filter(Boolean).join('\n');+}-## Resources+// eslint-disable-next-line @typescript-eslint/no-explicit-any+function formatPage(page: any) {+ return `- https://docs.composio.dev${page.url}.md`;+}-${getPage('debugging-info')}+export async function GET() {+ try {+ const docsTree = walkPageTree(source.pageTree.children as TreeNode[]);-### Migration Guide+ const examplesPages = examplesSource.getPages();+ const referencePages = referenceSource.getPages();+ const toolkitsPages = toolkitsSource.getPages();-${getFolderPages('migration-guide')}+ const index = `# Composio Documentation-### Troubleshooting+> Composio powers 800+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.-${getFolderPages('troubleshooting')}+${docsTree}## Examplesdiff --git a/docs/lib/source.ts b/docs/lib/source.tsindex 5544c62b1..ef6c8f2da 100644--- a/docs/lib/source.ts+++ b/docs/lib/source.ts@@ -306,7 +306,8 @@ function stripTwoslashFromCodeBlocks(content: string): string {});}-export async function getLLMText(page: InferPageType<typeof source>) {+export async function getLLMText(page: InferPageType<typeof source>, options?: { includeFooter?: boolean }) {+ const includeFooter = options?.includeFooter ?? true;// Fall back to description if getText is not availableif (typeof page.data.getText !== 'function') {return `# ${page.data.title} (${page.url})@@ -337,13 +338,13 @@ ${page.data.description || ''}`;const cleanContent = mdxToCleanMarkdown(content);- return `# ${page.data.title} (${page.url})--${cleanContent}+ const footer = includeFooter+ ? `\n\n---\n\n📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`+ : '';----+ return `# ${page.data.title} (${page.url})-📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`;+${cleanContent}${footer}`;}export function formatDate(dateStr: string): string {