Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix
mogery4e6cd03f4ec2
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 509a46fac..a053c2fc5 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -579,7 +579,10 @@ export async function buildFallbackList(meta: Meta): Promise<}}- if (!isXTwitterUrl(meta.url)) {+ if (isXTwitterUrl(meta.url) && _engines.includes("x-twitter")) {+ _engines.length = 0;+ _engines.push("x-twitter");+ } else if (!isXTwitterUrl(meta.url)) {const xTwitterIndex = _engines.indexOf("x-twitter");if (xTwitterIndex !== -1) {_engines.splice(xTwitterIndex, 1);diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex a19213d5f..66dc30e90 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -777,7 +777,12 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {break;} catch (error) {if (error instanceof WrappedEngineError) {- if (error.error instanceof EngineError) {+ if (error.engine === "x-twitter") {+ meta.logger.warn("X/Twitter scrape failed fatally.", {+ error: error.error,+ });+ throw error.error;+ } else if (error.error instanceof EngineError) {meta.logger.warn("Engine " + error.engine + " could not scrape the page.",{