Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
drop urlSpecificParams (#4755)
rowinsg8272df14f1c3
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c9649e6c2..36c264b3b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -64,7 +64,6 @@ import {import { ScrapeRetryTracker } from "./retryTracker";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";-import { urlSpecificParams } from "./lib/urlSpecificParams";import { shouldCheckRobots } from "./shouldCheckRobots";import { loadMock, MockState } from "./lib/mock";import { CostTracking } from "../../lib/cost-tracking";@@ -422,16 +421,6 @@ async function buildMetaObject(internalOptions: InternalOptions,costTracking: CostTracking,): Promise<Meta> {- const specParams =- urlSpecificParams[new URL(url).hostname.replace(/^www\./, "")];- if (specParams !== undefined) {- options = Object.assign(options, specParams.scrapeOptions);- internalOptions = Object.assign(- internalOptions,- specParams.internalOptions,- );- }-if (internalOptions.forceEngine === undefined) {const forcedEngine = getEngineForUrl(url);if (forcedEngine !== undefined) {diff --git a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts b/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.tsdeleted file mode 100644index 143d34cf0..000000000--- a/apps/api/src/scraper/scrapeURL/lib/urlSpecificParams.ts+++ /dev/null@@ -1,43 +0,0 @@-import { InternalOptions } from "..";-import { ScrapeOptions } from "../../../controllers/v1/types";--type UrlSpecificParams = {- scrapeOptions: Partial<ScrapeOptions>;- internalOptions: Partial<InternalOptions>;-};--// const docsParam: UrlSpecificParams = {-// scrapeOptions: { waitFor: 2000 },-// internalOptions: {},-// }--export const urlSpecificParams: Record<string, UrlSpecificParams> = {- // "support.greenpay.me": docsParam,- // "docs.pdw.co": docsParam,- // "developers.notion.com": docsParam,- // "docs2.hubitat.com": docsParam,- // "rsseau.fr": docsParam,- // "help.salesforce.com": docsParam,- // "scrapethissite.com": {- // scrapeOptions: {},- // internalOptions: { forceEngine: "fetch" },- // },- // "eonhealth.com": {- // defaultScraper: "fire-engine",- // params: {- // fireEngineOptions: {- // mobileProxy: true,- // method: "get",- // engine: "request",- // },- // },- // },- "digikey.com": {- scrapeOptions: {},- internalOptions: { forceEngine: "fire-engine;tlsclient" },- },- "lorealparis.hu": {- scrapeOptions: {},- internalOptions: { forceEngine: "fire-engine;tlsclient" },- },-};