Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: misc improvements
Nicolas3604f2a3ae8b
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex da12980cb..a50357d1c 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -156,13 +156,21 @@ export async function getMapResults({}// Parallelize sitemap index query with search results- const [sitemapIndexUrls, ...searchResults] = await Promise.all([+ const [sitemapIndexResult, ...searchResults] = await Promise.all([querySitemapIndex(url),...(cachedResult ? [] : pagePromises),]);- // Only query sitemap if index has less than 100 links- if (!ignoreSitemap && sitemapIndexUrls.length < 100) {+ const twoDaysAgo = new Date();+ twoDaysAgo.setDate(twoDaysAgo.getDate() - 2);+++ // If sitemap is not ignored and either we have few URLs (<100) or the data is stale (>2 days old), fetch fresh sitemap+ if (+ !ignoreSitemap &&+ (sitemapIndexResult.urls.length < 100 ||+ new Date(sitemapIndexResult.lastUpdated) < twoDaysAgo)+ ) {await crawler.tryGetSitemap(urls => {links.push(...urls);}, true, false, 30000);@@ -197,7 +205,7 @@ export async function getMapResults({}// Add sitemap-index URLs- links.push(...sitemapIndexUrls);+ links.push(...sitemapIndexResult.urls);// Perform cosine similarity between the search query and the list of linksif (search) {diff --git a/apps/api/src/scraper/WebScraper/sitemap-index.ts b/apps/api/src/scraper/WebScraper/sitemap-index.tsindex 4a97f4084..75d2532cd 100644--- a/apps/api/src/scraper/WebScraper/sitemap-index.ts+++ b/apps/api/src/scraper/WebScraper/sitemap-index.ts@@ -16,15 +16,20 @@ async function querySitemapIndexFunction(url: string) {try {const { data, error } = await supabase_service.from("crawl_maps")- .select("urls")- .eq("origin_url", originUrl);+ .select("urls, updated_at")+ .eq("origin_url", originUrl)+ .order("updated_at", { ascending: false });if (error) {throw error;}+ if (!data || data.length === 0) {+ return { urls: [], lastUpdated: new Date(0) };+ }+const allUrls = [...new Set(data.map((entry) => entry.urls).flat().map(url => normalizeUrl(url)))];- return allUrls;+ return { urls: allUrls, lastUpdated: data[0].updated_at };} catch (error) {logger.error("(sitemap-index) Error querying the index", {@@ -33,12 +38,12 @@ async function querySitemapIndexFunction(url: string) {});if (attempt === 3) {- return [];+ return { urls: [], lastUpdated: new Date(0) };}}}- return [];+ return { urls: [], lastUpdated: new Date(0) };}-export const querySitemapIndex = withAuth(querySitemapIndexFunction, []);+export const querySitemapIndex = withAuth(querySitemapIndexFunction, { urls: [], lastUpdated: new Date(0) });diff --git a/apps/api/src/services/indexing/crawl-maps-index.ts b/apps/api/src/services/indexing/crawl-maps-index.tsindex fb6388f58..35e377ea6 100644--- a/apps/api/src/services/indexing/crawl-maps-index.ts+++ b/apps/api/src/services/indexing/crawl-maps-index.ts@@ -91,7 +91,16 @@ async function processBatch() {const inserts: CrawlMapRecord[] = [];const duplicatesToDelete: string[] = [];+ // Track processed origins to avoid duplicates in the same batch+ const processedOrigins = new Set<string>();+for (const op of operations) {+ // Skip if we've already processed this origin in this batch+ if (processedOrigins.has(op.originUrl)) {+ continue;+ }+ processedOrigins.add(op.originUrl);+const existingForOrigin = mapsByOrigin.get(op.originUrl) || [];if (existingForOrigin.length > 0) {@@ -110,7 +119,7 @@ async function processBatch() {];updates.push({- id: mostRecent.id, // Add id to ensure we update the correct record+ id: mostRecent.id,origin_url: op.originUrl,urls: mergedUrls,num_urls: mergedUrls.length,@@ -156,14 +165,19 @@ async function processBatch() {logger.info(`🔄 Updating ${updates.length} existing crawl maps`, {origins: updates.map((u) => u.origin_url),});- const { error: updateError } = await supabase_service- .from("crawl_maps")- .upsert(updates);++ // Process updates one at a time to avoid conflicts+ for (const update of updates) {+ const { error: updateError } = await supabase_service+ .from("crawl_maps")+ .upsert(update);- if (updateError) {- logger.error("Failed to batch update crawl maps", {- error: updateError,- });+ if (updateError) {+ logger.error("Failed to update crawl map", {+ error: updateError,+ origin: update.origin_url+ });+ }}}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 83d1909b9..1e6964fa2 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -13,7 +13,7 @@ import osfrom .firecrawl import FirecrawlApp # noqa-__version__ = "1.10.1"+__version__ = "1.10.2"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex b0c09334f..04e90c39e 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -51,6 +51,11 @@ class FirecrawlApp:schema_: Optional[Any] = pydantic.Field(None, alias='schema')system_prompt: Optional[str] = Noneallow_external_links: Optional[bool] = False+ enable_web_search: Optional[bool] = False+ # Just for backwards compatibility+ enableWebSearch: Optional[bool] = False++class ExtractResponse(pydantic.BaseModel):"""