Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
timeouts for index engine
Gergő Móricz24a5a719210e
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 4c12c3601..2e37b2ba7 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -2,7 +2,7 @@ import { Document } from "../../../../controllers/v1/types";import { EngineScrapeResult } from "..";import { Meta } from "../..";import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob } from "../../../../services";-import { EngineError, IndexMissError } from "../../error";+import { EngineError, IndexMissError, TimeoutError } from "../../error";import crypto from "crypto";export async function sendDocumentToIndex(meta: Meta, document: Document) {@@ -133,7 +133,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {const errorCountToRegister = 3;-export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult> {+export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {const normalizedURL = normalizeURLForIndex(meta.url);const urlHash = hashURL(normalizedURL);@@ -162,11 +162,16 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResultselector = selector.is("location_languages", null);}- const { data, error } = await selector- .order("created_at", { ascending: false })- .limit(5);+ const { data, error } = await Promise.race([+ selector+ .order("created_at", { ascending: false })+ .limit(5),+ new Promise<{ data: { id: any; created_at: any; status: any }[], error: any }>((resolve, reject) => {+ setTimeout(() => reject(new TimeoutError()), timeToRun ?? 10000);+ }),+ ]);- if (error) {+ if (error || !data) {throw new EngineError("Failed to retrieve URL from DB index", {cause: error,});