Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update website_params.ts

Nicolasb8266cc32995
diff --git a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
index 659e4b889..89030fc38 100644
--- a/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
+++ b/apps/api/src/scraper/WebScraper/utils/custom/website_params.ts
@@ -1,13 +1,5 @@
export const urlSpecificParams = {
- "platform.openai.com": {
- defaultScraper: "fire-engine",
- params:{
- wait: 3000,
- fireEngineOptions:{
- engine: "chrome-cdp"
- },
- },
- },
+
"support.greenpay.me":{
defaultScraper: "fire-engine",
params: {
@@ -139,24 +131,6 @@ export const urlSpecificParams = {
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
},
},
- "firecrawl.dev":{
- defaultScraper: "fire-engine",
- params: {
- engine: "playwright",
- headers: {
- "User-Agent":
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
- "sec-fetch-site": "same-origin",
- "sec-fetch-mode": "cors",
- "sec-fetch-dest": "empty",
- referer: "https://www.google.com/",
- "accept-language": "en-US,en;q=0.9",
- "accept-encoding": "gzip, deflate, br",
- accept:
- "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
- },
- },
- },
"ir.veeva.com":{
defaultScraper: "fire-engine",
},