Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: llmstxt improvements
Nicolasacf1e60608f2
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex 55c06124f..bf28a00f6 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -28,6 +28,19 @@ const DescriptionSchema = z.object({title: z.string(),});+// Helper function to remove page separators+function removePageSeparators(text: string): string {+ return text.replace(/<\|firecrawl-page-\d+-lllmstxt\|>\n/g, '');+}++// Helper function to limit pages in full text+function limitPages(fullText: string, maxPages: number): string {+ const pages = fullText.split(/<\|firecrawl-page-\d+-lllmstxt\|>\n/);+ // First element is the header, so we start from index 1+ const limitedPages = pages.slice(0, maxPages + 1);+ return limitedPages.join('');+}+export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOptions) {const openai = new OpenAI();const { generationId, teamId, plan, url, maxUrls, showFullText, subId } = options;@@ -45,20 +58,23 @@ export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOptif (cachedResult) {logger.info("Found cached LLMs text", { url });+ // Limit pages and remove separators before returning+ const limitedFullText = limitPages(cachedResult.llmstxt_full, maxUrls);+ const cleanFullText = removePageSeparators(limitedFullText);+// Update final result with cached textawait updateGeneratedLlmsTxt(generationId, {status: "completed",generatedText: cachedResult.llmstxt,- fullText: cachedResult.llmstxt_full,+ fullText: cleanFullText,showFullText: showFullText,});-return {success: true,data: {generatedText: cachedResult.llmstxt,- fullText: cachedResult.llmstxt_full,+ fullText: cleanFullText,showFullText: showFullText,},};@@ -144,25 +160,29 @@ export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOptif (!result) continue;llmstxt += `- [${result.title}](${result.url}): ${result.description}\n`;- llmsFulltxt += `## ${result.title}\n${result.markdown}\n\n`;+ llmsFulltxt += `<|firecrawl-page-${i + batchResults.indexOf(result) + 1}-lllmstxt|>\n## ${result.title}\n${result.markdown}\n\n`;}// Update progress after each batchawait updateGeneratedLlmsTxt(generationId, {status: "processing",generatedText: llmstxt,- fullText: llmsFulltxt,+ fullText: removePageSeparators(llmsFulltxt),});}// After successful generation, save to cacheawait saveLlmsTextToCache(url, llmstxt, llmsFulltxt, maxUrls);+ // Limit pages and remove separators before final update+ const limitedFullText = limitPages(llmsFulltxt, maxUrls);+ const cleanFullText = removePageSeparators(limitedFullText);+// Update final result with both generated text and full textawait updateGeneratedLlmsTxt(generationId, {status: "completed",generatedText: llmstxt,- fullText: llmsFulltxt,+ fullText: cleanFullText,showFullText: showFullText,});@@ -197,7 +217,7 @@ export async function performGenerateLlmsTxt(options: GenerateLLMsTextServiceOptsuccess: true,data: {generatedText: llmstxt,- fullText: llmsFulltxt,+ fullText: cleanFullText,showFullText: showFullText,},};diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.tsindex 73be813b8..4120170a2 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts@@ -33,6 +33,14 @@ export async function getLlmsTextFromCache(return null;}+ // Check if data is older than 1 week+ const oneWeekAgo = new Date();+ oneWeekAgo.setDate(oneWeekAgo.getDate() - 7);++ if (!data || new Date(data.updated_at) < oneWeekAgo) {+ return null;+ }+return data;} catch (error) {logger.error("Failed to fetch LLMs text from cache", { error, originUrl });@@ -53,28 +61,47 @@ export async function saveLlmsTextToCache(const originUrl = normalizeUrlOnlyHostname(url);try {- // First check if there's an existing entry with fewer URLs+ // First check if there's an existing entryconst { data: existingData } = await supabase_service.from("llm_texts").select("*").eq("origin_url", originUrl).single();- // Always update the entry for the origin URL- const { error } = await supabase_service- .from("llm_texts")- .update({- llmstxt,- llmstxt_full,- max_urls: maxUrls,- updated_at: new Date().toISOString(),- })- .eq("origin_url", originUrl);+ if (existingData) {+ // Update existing entry+ const { error } = await supabase_service+ .from("llm_texts")+ .update({+ llmstxt,+ llmstxt_full,+ max_urls: maxUrls,+ updated_at: new Date().toISOString(),+ })+ .eq("origin_url", originUrl);- if (error) {- logger.error("Error saving LLMs text to cache", { error, originUrl });+ if (error) {+ logger.error("Error updating LLMs text in cache", { error, originUrl });+ } else {+ logger.debug("Successfully updated cached LLMs text", { originUrl, maxUrls });+ }} else {- logger.debug("Successfully cached LLMs text", { originUrl, maxUrls });+ // Insert new entry+ const { error } = await supabase_service+ .from("llm_texts")+ .insert({+ origin_url: originUrl,+ llmstxt,+ llmstxt_full,+ max_urls: maxUrls,+ updated_at: new Date().toISOString(),+ });++ if (error) {+ logger.error("Error inserting LLMs text to cache", { error, originUrl });+ } else {+ logger.debug("Successfully inserted new cached LLMs text", { originUrl, maxUrls });+ }}} catch (error) {logger.error("Failed to save LLMs text to cache", { error, originUrl });