Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update website_params.ts
Nicolasb8266cc32995
diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.tsindex 659e4b889..89030fc38 100644--- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts+++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts@@ -1,13 +1,5 @@export const urlSpecificParams = {- "platform.openai.com": {- defaultScraper: "fire-engine",- params:{- wait: 3000,- fireEngineOptions:{- engine: "chrome-cdp"- },- },- },+"support.greenpay.me":{defaultScraper: "fire-engine",params: {@@ -139,24 +131,6 @@ export const urlSpecificParams = {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",},},- "firecrawl.dev":{- defaultScraper: "fire-engine",- params: {- engine: "playwright",- headers: {- "User-Agent":- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",- "sec-fetch-site": "same-origin",- "sec-fetch-mode": "cors",- "sec-fetch-dest": "empty",- referer: "https://www.google.com/",- "accept-language": "en-US,en;q=0.9",- "accept-encoding": "gzip, deflate, br",- accept:- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",- },- },- },"ir.veeva.com":{defaultScraper: "fire-engine",},