Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

timeouts for index engine

Gergő Móricz24a5a719210e
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 4c12c3601..2e37b2ba7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -2,7 +2,7 @@ import { Document } from "../../../../controllers/v1/types";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob } from "../../../../services";
-import { EngineError, IndexMissError } from "../../error";
+import { EngineError, IndexMissError, TimeoutError } from "../../error";
import crypto from "crypto";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
@@ -133,7 +133,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
const errorCountToRegister = 3;
-export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult> {
+export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {
const normalizedURL = normalizeURLForIndex(meta.url);
const urlHash = hashURL(normalizedURL);
@@ -162,11 +162,16 @@ export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult
selector = selector.is("location_languages", null);
}
- const { data, error } = await selector
- .order("created_at", { ascending: false })
- .limit(5);
+ const { data, error } = await Promise.race([
+ selector
+ .order("created_at", { ascending: false })
+ .limit(5),
+ new Promise<{ data: { id: any; created_at: any; status: any }[], error: any }>((resolve, reject) => {
+ setTimeout(() => reject(new TimeoutError()), timeToRun ?? 10000);
+ }),
+ ]);
- if (error) {
+ if (error || !data) {
throw new EngineError("Failed to retrieve URL from DB index", {
cause: error,
});