Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixes
Nicolas52f2e733e2a8
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 86f798e96..1f4fd8a82 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -6,7 +6,7 @@"scripts": {"start": "nodemon --exec ts-node src/index.ts","start:production": "tsc && node dist/src/index.js",- "format": "npx prettier --write \"src/**/*.(js|ts)\"",+ "format": "prettier --write \"src/**/*.(js|ts)\"","flyio": "node dist/src/index.js","start:dev": "nodemon --exec ts-node src/index.ts","build": "tsc && pnpm sentry:sourcemaps",@@ -102,6 +102,7 @@"pdf-parse": "^1.1.1","pos": "^0.4.2","posthog-node": "^4.0.1",+ "prettier": "^3.4.2","promptable": "^0.0.10","puppeteer": "^22.12.1","rate-limiter-flexible": "2.4.2",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 4557afa99..563965c1e 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -164,6 +164,9 @@ importers:posthog-node:specifier: ^4.0.1version: 4.0.1+ prettier:+ specifier: ^3.4.2+ version: 3.4.2promptable:specifier: ^0.0.10version: 0.0.10@@ -3756,6 +3759,11 @@ packages:resolution: {integrity: sha512-rtqm2h22QxLGBrW2bLYzbRhliIrqgZ0k+gF0LkQ1SNdeD06YE5eilV0MxZppFSxC8TfH0+B0cWCuebEnreIDgQ==}engines: {node: '>=15.0.0'}+ resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==}+ engines: {node: '>=14'}+ hasBin: true+resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==}engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}@@ -4321,8 +4329,8 @@ packages:engines: {node: '>=14.17'}hasBin: true- resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==}+ resolution: {integrity: sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==}engines: {node: '>=14.17'}hasBin: true@@ -8978,6 +8986,8 @@ snapshots:transitivePeerDependencies:- debug+ [email protected]: {}+dependencies:'@jest/schemas': 29.6.3@@ -9000,7 +9010,7 @@ snapshots:csv-parse: 5.5.6gpt3-tokenizer: 1.1.5openai: 3.3.0- typescript: 5.6.3+ typescript: 5.7.2uuid: 9.0.1zod: 3.23.8transitivePeerDependencies:@@ -9584,7 +9594,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}[email protected](@babel/[email protected]):dependencies:diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex b8c6bc630..bb9ba3639 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -86,12 +86,10 @@ export async function crawlController(req: Request, res: Response) {} = await checkTeamCredits(chunk, team_id, limitCheck);if (!creditsCheckSuccess) {- return res- .status(402)- .json({- error:- "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at [email protected]"- });+ return res.status(402).json({+ error:+ "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at [email protected]"+ });}// TODO: need to do this to v1@@ -259,10 +257,8 @@ export async function crawlController(req: Request, res: Response) {} catch (error) {Sentry.captureException(error);logger.error(error);- return res- .status(500)- .json({- error: error instanceof ZodError ? "Invalid URL" : error.message- });+ return res.status(500).json({+ error: error instanceof ZodError ? "Invalid URL" : error.message+ });}}diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex c7c8d9fe6..4a761ea36 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -211,12 +211,10 @@ export async function scrapeController(req: Request, res: Response) {await checkTeamCredits(chunk, team_id, 1);if (!creditsCheckSuccess) {earlyReturn = true;- return res- .status(402)- .json({- error:- "Insufficient credits. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing"- });+ return res.status(402).json({+ error:+ "Insufficient credits. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing"+ });}} catch (error) {logger.error(error);diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 905c32d8e..a4f4445b8 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -235,14 +235,12 @@ app.use(" -- " +verbose);- res- .status(500)- .json({- success: false,- error:- "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " +- id- });+ res.status(500).json({+ success: false,+ error:+ "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " ++ id+ });});diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 206423baa..a9727e00f 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -54,13 +54,11 @@ function checkCreditsMiddleware(`Insufficient credits: ${JSON.stringify({ team_id: req.auth.team_id, minimum, remainingCredits })}`);if (!res.headersSent) {- return res- .status(402)- .json({- success: false,- error:- "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing or try changing the request limit to a lower value."- });+ return res.status(402).json({+ success: false,+ error:+ "Insufficient credits to perform this request. For more credits, you can upgrade your plan at https://firecrawl.dev/pricing or try changing the request limit to a lower value."+ });}}req.account = { remainingCredits };@@ -122,13 +120,11 @@ function idempotencyMiddleware(function blocklistMiddleware(req: Request, res: Response, next: NextFunction) {if (typeof req.body.url === "string" && isUrlBlocked(req.body.url)) {if (!res.headersSent) {- return res- .status(403)- .json({- success: false,- error:- "URL is blocked intentionally. Firecrawl currently does not support social media scraping due to policy restrictions."- });+ return res.status(403).json({+ success: false,+ error:+ "URL is blocked intentionally. Firecrawl currently does not support social media scraping due to policy restrictions."+ });}}next();