Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(docs): clean MDX content for AI agents
Sushmithamalleshc66fa32ba838
diff --git a/docs/app/llms-full.txt/route.ts b/docs/app/llms-full.txt/route.tsindex 187b91067..9fe109265 100644--- a/docs/app/llms-full.txt/route.ts+++ b/docs/app/llms-full.txt/route.ts@@ -1,23 +1,13 @@-import { source } from '@/lib/source';+import { getLLMText, source } from '@/lib/source';export const revalidate = false;export async function GET() {try {const pages = source.getPages();- const results: string[] = [];+ const results = await Promise.all(pages.map(getLLMText));- for (const page of pages) {- try {- const processed = await page.data.getText('processed');- results.push(`# ${page.data.title}\n\n${processed}`);- } catch {- // Fallback to basic info if getText fails- results.push(`# ${page.data.title}\n\n${page.data.description || ''}`);- }- }-- return new Response(results.join('\n\n'), {+ return new Response(results.join('\n\n---\n\n'), {headers: {'Content-Type': 'text/plain; charset=utf-8',},diff --git a/docs/lib/source.ts b/docs/lib/source.tsindex b24069dd6..c00223372 100644--- a/docs/lib/source.ts+++ b/docs/lib/source.ts@@ -55,12 +55,79 @@ export function getPageImage(page: InferPageType<typeof source>) {};}+/**+ * Converts MDX content to clean markdown for AI agents.+ * Strips JSX components and converts them to plain text equivalents.+ */+function mdxToCleanMarkdown(content: string): string {+ let result = content;++ // Remove frontmatter+ result = result.replace(/^---[\s\S]*?---\n*/m, '');++ // Convert YouTube to link+ result = result.replace(+ /<YouTube\s+id="([^"]+)"\s+title="([^"]+)"\s*\/>/g,+ '[Video: $2](https://youtube.com/watch?v=$1)'+ );++ // Convert Callout to blockquote+ result = result.replace(+ /<Callout[^>]*title="([^"]*)"[^>]*>([\s\S]*?)<\/Callout>/g,+ '> **$1**: $2'+ );+ result = result.replace(+ /<Callout[^>]*>([\s\S]*?)<\/Callout>/g,+ '> $1'+ );++ // Convert Card - handle multiline and various attribute orders+ result = result.replace(+ /<Card[\s\S]*?title="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?\/>/g,+ '- [$1]($2): $3'+ );++ // Convert ProviderCard to markdown link+ result = result.replace(+ /<ProviderCard[\s\S]*?name="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?languages=\{\[([^\]]*)\]\}[\s\S]*?\/>/g,+ (_, name, href, langs) => `- [${name}](${href}) (${langs.replace(/"/g, '')})`+ );++ // Convert Tabs/Tab content - keep inner content+ result = result.replace(/<TabsList>[\s\S]*?<\/TabsList>/g, '');+ result = result.replace(/<TabsTrigger[^>]*>[^<]*<\/TabsTrigger>/g, '');+ result = result.replace(/<TabsContent[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/TabsContent>/g, '\n**$1:**\n$2');+ result = result.replace(/<Tab[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Tab>/g, '\n**$1:**\n$2');++ // Convert Steps/Step+ result = result.replace(/<Step>\s*###\s*(.+)/g, '### Step: $1');+ result = result.replace(/<\/?Steps?>/g, '');++ // Remove wrapper components (Cards, ProviderGrid, Tabs, Frame, div, etc.)+ result = result.replace(/<\/?(Cards|ProviderGrid|Tabs|Frame|div)[^>]*>/g, '');++ // Remove remaining self-closing JSX tags (including those with JSX expressions)+ result = result.replace(/<[A-Z][a-zA-Z]*[\s\S]*?\/>/g, '');++ // Remove remaining JSX opening/closing tags but keep content+ result = result.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, '');++ // Clean up leftover JSX artifacts like lone } or {+ result = result.replace(/^\s*[{}]\s*$/gm, '');++ // Clean up excessive newlines+ result = result.replace(/\n{3,}/g, '\n\n');++ return result.trim();+}+export async function getLLMText(page: InferPageType<typeof source>) {- const processed = await page.data.getText('processed');+ const content = await page.data.getText('raw');+ const cleanContent = mdxToCleanMarkdown(content);return `# ${page.data.title} (${page.url})-${processed}`;+${cleanContent}`;}export function formatDate(dateStr: string): string {