Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: bug fixes + acuc fixes + cache fixes
Nicolas04916f17e2b8
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 16622bb0d..22c752320 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -105,6 +105,7 @@ export async function getACUC({ get: true },));+if (!error) {break;}@@ -139,8 +140,17 @@ export async function getACUC(}export async function clearACUC(api_key: string): Promise<void> {- const cacheKeyACUC = `acuc_${api_key}`;- await deleteKey(cacheKeyACUC);+ // Delete cache for all rate limiter modes+ const modes = Object.values(RateLimiterMode);+ await Promise.all(+ modes.map(async (mode) => {+ const cacheKey = `acuc_${api_key}_${mode}`;+ await deleteKey(cacheKey);+ })+ );++ // Also clear the base cache key+ await deleteKey(`acuc_${api_key}`);}export async function authenticateUser(@@ -188,7 +198,7 @@ export async function supaAuthenticateUser(let teamId: string | null = null;let priceId: string | null = null;let chunk: AuthCreditUsageChunk | null = null;-+ let plan: PlanType = "free";if (token == "this_is_just_a_preview_token") {if (mode == RateLimiterMode.CrawlStatus) {rateLimiter = getRateLimiter(RateLimiterMode.CrawlStatus, token);@@ -198,6 +208,7 @@ export async function supaAuthenticateUser(rateLimiter = getRateLimiter(RateLimiterMode.Preview, token);}teamId = "preview";+ plan = "free";} else {normalizedApi = parseApi(token);if (!normalizedApiIsUuid(normalizedApi)) {@@ -221,7 +232,8 @@ export async function supaAuthenticateUser(teamId = chunk.team_id;priceId = chunk.price_id;- const plan = getPlanByPriceId(priceId);++ plan = getPlanByPriceId(priceId);subscriptionData = {team_id: teamId,plan,@@ -322,7 +334,7 @@ export async function supaAuthenticateUser(mode === RateLimiterMode.Extract ||mode === RateLimiterMode.Search)) {- return { success: true, team_id: "preview", chunk: null };+ return { success: true, team_id: "preview", chunk: null, plan: "free" };// check the origin of the request and make sure its from firecrawl.dev// const origin = req.headers.origin;// if (origin && origin.includes("firecrawl.dev")){@@ -369,6 +381,15 @@ function getPlanByPriceId(price_id: string | null): PlanType {case process.env.STRIPE_PRICE_ID_ETIER_SCALE_1_MONTHLY:case process.env.STRIPE_PRICE_ID_ETIER_SCALE_1_YEARLY:return "etierscale1";+ case process.env.STRIPE_PRICE_ID_EXTRACT_STARTER_MONTHLY:+ case process.env.STRIPE_PRICE_ID_EXTRACT_STARTER_YEARLY:+ return "extract_starter";+ case process.env.STRIPE_PRICE_ID_EXTRACT_EXPLORER_MONTHLY:+ case process.env.STRIPE_PRICE_ID_EXTRACT_EXPLORER_YEARLY:+ return "extract_explorer";+ case process.env.STRIPE_PRICE_ID_EXTRACT_PRO_MONTHLY:+ case process.env.STRIPE_PRICE_ID_EXTRACT_PRO_YEARLY:+ return "extract_pro";default:return "free";}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex c1dc4c047..b4eff93f6 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -17,6 +17,7 @@ import expressWs from "express-ws";import { ErrorResponse, ResponseWithSentry } from "./controllers/v1/types";import { ZodError } from "zod";import { v4 as uuidv4 } from "uuid";+import { RateLimiterMode } from "./types";const { createBullBoard } = require("@bull-board/api");const { BullAdapter } = require("@bull-board/api/bullAdapter");@@ -249,7 +250,6 @@ app.use();logger.info(`Worker ${process.pid} started`);-// const sq = getScrapeQueue();// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));diff --git a/apps/api/src/lib/cache.ts b/apps/api/src/lib/cache.tsindex 85e57e51d..611bc043a 100644--- a/apps/api/src/lib/cache.ts+++ b/apps/api/src/lib/cache.ts@@ -41,6 +41,11 @@ export type CacheEntry = {export async function saveEntryToCache(key: string, entry: CacheEntry) {if (!cacheRedis) return;+ if (!entry.html || entry.html.length < 100) {+ logger.warn("Skipping cache save for short HTML", { key, htmlLength: entry.html?.length });+ return;+ }+try {await cacheRedis.set(key, JSON.stringify(entry), "EX", 3600); // 1 hour in seconds} catch (error) {diff --git a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts b/apps/api/src/scraper/scrapeURL/engines/cache/index.tsindex c0451df4e..31075f920 100644--- a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/cache/index.ts@@ -10,6 +10,14 @@ export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {const entry = await getEntryFromCache(key);if (entry === null) throw new EngineError("Cache missed");+ if (!entry.html) {+ throw new EngineError("Cache hit but HTML is missing");+ }+ if (entry.html.length < 100) {+ throw new EngineError("Cache hit but HTML is too short to be useful");+ }++// Set fromCache flag to indicate this document was retrieved from cachemeta.internalOptions.fromCache = true;diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 09c8d7493..68cea8faa 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -19,6 +19,10 @@ const RATE_LIMITS = {etier1a: 1000,etier2a: 300,etierscale1: 150,+ // extract ops+ extract_starter: 20,+ extract_explorer: 100,+ extract_pro: 1000,},scrape: {default: 20,@@ -36,6 +40,10 @@ const RATE_LIMITS = {etier1a: 1000,etier2a: 2500,etierscale1: 1500,+ // extract ops+ extract_starter: 20,+ extract_explorer: 100,+ extract_pro: 1000,},search: {default: 20,@@ -53,6 +61,10 @@ const RATE_LIMITS = {etier1a: 1000,etier2a: 2500,etierscale1: 1500,+ // extract ops+ extract_starter: 20,+ extract_explorer: 100,+ extract_pro: 1000,},map: {default: 20,@@ -70,6 +82,10 @@ const RATE_LIMITS = {etier1a: 1000,etier2a: 2500,etierscale1: 1500,+ // extract ops+ extract_starter: 20,+ extract_explorer: 100,+ extract_pro: 1000,},extract: {default: 20,@@ -87,6 +103,9 @@ const RATE_LIMITS = {etier1a: 1000,etier2a: 1000,etierscale1: 1000,+ extract_starter: 20,+ extract_explorer: 100,+ extract_pro: 1000,},preview: {free: 5,diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex ce535ede9..a894c3293 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -186,6 +186,9 @@ export type PlanType =| "etier1a"| "etierscale1"| "free"+ | "extract_starter"+ | "extract_explorer"+ | "extract_pro"| "";export type WebhookEventType =