Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): gate llmstxt on credits and cache it per URL (#5651)
tomsideguide8f3694603665
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.tsindex a1e1c70fa..253469438 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts@@ -3,7 +3,7 @@ import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { config } from "../../config";import { logger } from "../logger";-import { normalizeUrl, normalizeUrlOnlyHostname } from "../canonical-url";+import { normalizeUrl } from "../canonical-url";interface LlmsTextCache {origin_url: string;@@ -20,7 +20,7 @@ export async function getLlmsTextFromCache(return null;}- const originUrl = normalizeUrlOnlyHostname(url);+ const originUrl = normalizeUrl(url);try {const [data] = await dbRr@@ -60,7 +60,7 @@ export async function saveLlmsTextToCache(return;}- const originUrl = normalizeUrlOnlyHostname(url);+ const originUrl = normalizeUrl(url);try {// First check if there's an existing entrydiff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 32f7305f7..a731dada9 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -174,6 +174,7 @@ v1Router.post(authMiddleware(RateLimiterMode.Scrape),deprecationMiddleware("v1_llmstxt"),restrictedOptionsCountryCheck,+ checkCreditsMiddleware(1),blocklistMiddleware,wrap(generateLLMsTextController),);