Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): gate llmstxt on credits and cache it per URL (#5651)

tomsideguide8f3694603665
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
index a1e1c70fa..253469438 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-supabase.ts
@@ -3,7 +3,7 @@ import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { config } from "../../config";
import { logger } from "../logger";
-import { normalizeUrl, normalizeUrlOnlyHostname } from "../canonical-url";
+import { normalizeUrl } from "../canonical-url";
interface LlmsTextCache {
origin_url: string;
@@ -20,7 +20,7 @@ export async function getLlmsTextFromCache(
return null;
}
- const originUrl = normalizeUrlOnlyHostname(url);
+ const originUrl = normalizeUrl(url);
try {
const [data] = await dbRr
@@ -60,7 +60,7 @@ export async function saveLlmsTextToCache(
return;
}
- const originUrl = normalizeUrlOnlyHostname(url);
+ const originUrl = normalizeUrl(url);
try {
// First check if there's an existing entry
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 32f7305f7..a731dada9 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -174,6 +174,7 @@ v1Router.post(
authMiddleware(RateLimiterMode.Scrape),
deprecationMiddleware("v1_llmstxt"),
restrictedOptionsCountryCheck,
+ checkCreditsMiddleware(1),
blocklistMiddleware,
wrap(generateLLMsTextController),
);