Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

perf: cache parsed toolkits.json in-process with O(1) slug lookup

Sushmithamallesh0bf5bee16580
diff --git a/docs/app/(home)/toolkits/[[...slug]]/page.tsx b/docs/app/(home)/toolkits/[[...slug]]/page.tsx
index 71ba47291..c63ae8b4d 100644
--- a/docs/app/(home)/toolkits/[[...slug]]/page.tsx
+++ b/docs/app/(home)/toolkits/[[...slug]]/page.tsx
@@ -6,12 +6,13 @@ import { ToolkitsLanding } from '@/components/toolkits/toolkits-landing';
import { PageActions } from '@/components/page-actions';
import { readFile } from 'fs/promises';
import { join } from 'path';
+import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
import { unified } from 'unified';
import remarkParse from 'remark-parse';
import remarkRehype from 'remark-rehype';
import { toHtml } from 'hast-util-to-html';
import type { Metadata } from 'next';
-import type { Toolkit, Tool, Trigger } from '@/types/toolkit';
+import type { Tool, Trigger } from '@/types/toolkit';
import type { FaqItem } from '@/components/toolkits/faq-section';
import { processSchema, toolFromApi } from '@/lib/toolkit-schema';
@@ -129,34 +130,6 @@ async function readToolkitFaq(slug: string): Promise<FaqItem[] | null> {
return items.length > 0 ? items : null;
}
-async function getToolkits(): Promise<Toolkit[]> {
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
-
- try {
- const data = await readFile(filePath, 'utf-8');
- const toolkits = JSON.parse(data) as Toolkit[];
-
- if (!Array.isArray(toolkits)) {
- throw new Error('toolkits.json must contain an array');
- }
-
- if (toolkits.length === 0) {
- console.warn('[Toolkits] Warning: toolkits.json is empty');
- }
-
- return toolkits;
- } catch (error) {
- const err = error as NodeJS.ErrnoException;
- if (err.code === 'ENOENT') {
- throw new Error(`Toolkits data file not found: ${filePath}`);
- }
- if (error instanceof SyntaxError) {
- throw new Error(`Invalid JSON in toolkits.json: ${error.message}`);
- }
- throw error;
- }
-}
-
export async function generateStaticParams() {
// Index page
const indexParam = { slug: [] };
@@ -165,7 +138,7 @@ export async function generateStaticParams() {
const mdxParams = toolkitsSource.generateParams();
// JSON toolkit pages
- const toolkits = await getToolkits();
+ const toolkits = await getAllToolkits();
const jsonParams = toolkits.map((toolkit) => ({
slug: [toolkit.slug],
}));
@@ -203,8 +176,7 @@ export async function generateMetadata({ params }: { params: Promise<{ slug?: st
// Check JSON toolkit
if (slug.length === 1) {
- const toolkits = await getToolkits();
- const toolkit = toolkits.find((t) => t.slug === slug[0]);
+ const toolkit = await getToolkitBySlug(slug[0]);
if (toolkit) {
const title = `${toolkit.name?.trim() || toolkit.slug} - Composio Toolkit`;
const description = `Build an AI agent that connects to ${toolkit.name?.trim() || toolkit.slug} using Composio. ${toolkit.description}`;
@@ -245,8 +217,7 @@ export default async function ToolkitsPage({ params }: { params: Promise<{ slug?
// Check JSON toolkit
if (slug.length === 1) {
const toolkitSlug = slug[0];
- const toolkits = await getToolkits();
- const toolkit = toolkits.find((t) => t.slug === toolkitSlug);
+ const toolkit = await getToolkitBySlug(toolkitSlug);
if (toolkit) {
// Fetch detailed tool/trigger info and FAQ content in parallel
diff --git a/docs/app/llms.mdx/[[...slug]]/route.ts b/docs/app/llms.mdx/[[...slug]]/route.ts
index eb732be08..eba1ce482 100644
--- a/docs/app/llms.mdx/[[...slug]]/route.ts
+++ b/docs/app/llms.mdx/[[...slug]]/route.ts
@@ -13,6 +13,7 @@ import { openapi } from '@/lib/openapi';
import { notFound } from 'next/navigation';
import { readFile } from 'fs/promises';
import { join } from 'path';
+import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
import type { Toolkit, Tool, Trigger, ParameterSchema } from '@/types/toolkit';
import { processSchema, toolFromApi } from '@/lib/toolkit-schema';
@@ -696,27 +697,6 @@ function toolkitToMarkdown(toolkit: Toolkit, detailedTools?: Tool[], detailedTri
return lines.join('\n');
}
-async function getToolkit(slug: string): Promise<Toolkit | null> {
- try {
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
- const data = await readFile(filePath, 'utf-8');
- const toolkits = JSON.parse(data) as Toolkit[];
- return toolkits.find((t) => t.slug === slug) || null;
- } catch {
- return null;
- }
-}
-
-async function getAllToolkits(): Promise<Toolkit[]> {
- try {
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
- const data = await readFile(filePath, 'utf-8');
- return JSON.parse(data) as Toolkit[];
- } catch {
- return [];
- }
-}
-
// Generate a comprehensive toolkits index for /toolkits.md
async function generateToolkitsIndex(): Promise<string> {
const toolkits = await getAllToolkits();
@@ -878,7 +858,7 @@ export async function GET(
// Special handling for JSON toolkit pages
if (prefix === 'toolkits' && rest.length === 1) {
- const toolkit = await getToolkit(rest[0]);
+ const toolkit = await getToolkitBySlug(rest[0]);
if (toolkit) {
// Fetch detailed tool/trigger info and FAQ content in parallel
const [detailedTools, detailedTriggers, faqMarkdown] = await Promise.all([
diff --git a/docs/app/sitemap.ts b/docs/app/sitemap.ts
index f8baf10d9..2f159863b 100644
--- a/docs/app/sitemap.ts
+++ b/docs/app/sitemap.ts
@@ -1,6 +1,4 @@
import type { MetadataRoute } from 'next';
-import { readFileSync } from 'fs';
-import { join } from 'path';
import {
source,
getReferenceSource,
@@ -9,41 +7,10 @@ import {
changelogEntries,
dateToChangelogUrl,
} from '@/lib/source';
+import { getAllToolkitsSync } from '@/lib/toolkit-data';
const baseUrl = 'https://docs.composio.dev';
-interface Toolkit {
- slug: string;
-}
-
-function getToolkitsFromJson(): Toolkit[] {
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
-
- try {
- const data = readFileSync(filePath, 'utf-8');
- const toolkits = JSON.parse(data) as Toolkit[];
-
- if (!Array.isArray(toolkits)) {
- throw new Error('toolkits.json must contain an array');
- }
-
- if (toolkits.length === 0) {
- console.warn('[Sitemap] Warning: toolkits.json is empty - toolkit pages will be missing from sitemap');
- }
-
- return toolkits;
- } catch (error) {
- const err = error as NodeJS.ErrnoException;
- if (err.code === 'ENOENT') {
- throw new Error(`[Sitemap] Toolkits data file not found: ${filePath}`);
- }
- if (error instanceof SyntaxError) {
- throw new Error(`[Sitemap] Invalid JSON in toolkits.json: ${error.message}`);
- }
- throw error;
- }
-}
-
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
const docsPages = source.getPages().map((page) => ({
url: `${baseUrl}${page.url}`,
@@ -65,7 +32,7 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
}));
// JSON toolkit pages (dynamically generated from toolkits.json)
- const toolkitsJsonPages = getToolkitsFromJson().map((toolkit) => ({
+ const toolkitsJsonPages = getAllToolkitsSync().map((toolkit) => ({
url: `${baseUrl}/toolkits/${toolkit.slug}`,
}));
diff --git a/docs/lib/toolkit-data.ts b/docs/lib/toolkit-data.ts
new file mode 100644
index 000000000..350414f3a
--- /dev/null
+++ b/docs/lib/toolkit-data.ts
@@ -0,0 +1,48 @@
+import { readFile } from 'fs/promises';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import type { Toolkit } from '@/types/toolkit';
+
+const TOOLKITS_PATH = join(process.cwd(), 'public/data/toolkits.json');
+
+let cached: { list: Toolkit[]; bySlug: Map<string, Toolkit> } | null = null;
+
+function buildCache(toolkits: Toolkit[]) {
+ cached = {
+ list: toolkits,
+ bySlug: new Map(toolkits.map(t => [t.slug, t])),
+ };
+ return cached;
+}
+
+function parse(data: string): Toolkit[] {
+ const toolkits = JSON.parse(data) as Toolkit[];
+
+ if (!Array.isArray(toolkits)) {
+ throw new Error('toolkits.json must contain an array');
+ }
+
+ if (toolkits.length === 0) {
+ console.warn('[Toolkits] Warning: toolkits.json is empty');
+ }
+
+ return toolkits;
+}
+
+export async function getAllToolkits(): Promise<Toolkit[]> {
+ if (cached) return cached.list;
+ const data = await readFile(TOOLKITS_PATH, 'utf-8');
+ return buildCache(parse(data)).list;
+}
+
+export async function getToolkitBySlug(slug: string): Promise<Toolkit | null> {
+ if (cached) return cached.bySlug.get(slug) ?? null;
+ const data = await readFile(TOOLKITS_PATH, 'utf-8');
+ return buildCache(parse(data)).bySlug.get(slug) ?? null;
+}
+
+export function getAllToolkitsSync(): Toolkit[] {
+ if (cached) return cached.list;
+ const data = readFileSync(TOOLKITS_PATH, 'utf-8');
+ return buildCache(parse(data)).list;
+}