Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Block bounty publication for restricted domains (#4570)
Developers Digestcd0a25445021
diff --git a/apps/api/src/routes/exchange-blocklist.ts b/apps/api/src/routes/exchange-blocklist.tsnew file mode 100644index 000000000..f1ed9fb13--- /dev/null+++ b/apps/api/src/routes/exchange-blocklist.ts@@ -0,0 +1,67 @@+import type { NextFunction, Request, Response } from "express";+import { parse } from "tldts";+import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";+import type { RequestWithAuth } from "../controllers/v1/types";++export function bountyDomains(body: Record<string, unknown>): string[] {+ const fields = [+ body.title,+ body.description,+ ...(Array.isArray(body.requirements) ? body.requirements : []),+ ];+ const domains = new Set<string>();+ for (const field of fields) {+ if (typeof field !== "string") continue;+ const candidates = [+ ...field.matchAll(/https?:\/\/[^\s<>"']+/giu),+ ...field.matchAll(+ /(?:[\p{L}\p{N}](?:[\p{L}\p{N}-]*[\p{L}\p{N}])?\.)+[\p{L}][\p{L}\p{N}-]*/gu,+ ),+ ];+ for (const candidate of candidates) {+ try {+ const url = new URL(+ /^https?:\/\//i.test(candidate[0])+ ? candidate[0]+ : `https://${candidate[0]}`,+ );+ const hostname = url.hostname.toLowerCase().replace(/\.$/, "");+ if (parse(hostname).domain) domains.add(hostname);+ } catch {}+ }+ }+ return [...domains];+}++export function bountyBlocklistMiddleware(+ req: Request,+ res: Response,+ next: NextFunction,+) {+ try {+ const request = req as RequestWithAuth<any, any, any>;+ const blocked = bountyDomains(req.body ?? {}).filter(domain =>+ isUrlBlocked(+ `https://${domain}/`,+ {},+ {+ team_id: request.auth.team_id,+ org_id: request.acuc?.org_id ?? null,+ origin: "exchange-bounty",+ },+ ),+ );+ if (blocked.length) {+ return res.status(403).json({+ success: false,+ error: {+ code: "bounty_domain_blocked",+ message: `Bounties cannot be published for blocked domains: ${blocked.join(", ")}.`,+ },+ });+ }+ next();+ } catch (error) {+ next(error);+ }+}diff --git a/apps/api/src/routes/exchange.ts b/apps/api/src/routes/exchange.tsindex 139cabcc1..0dbc4ab03 100644--- a/apps/api/src/routes/exchange.ts+++ b/apps/api/src/routes/exchange.ts@@ -1,3 +1,4 @@+import { bountyBlocklistMiddleware } from "./exchange-blocklist";import express, { Request, Response } from "express";import { Agent, fetch } from "undici";import { config } from "../config";@@ -171,6 +172,14 @@ exchangeRouter.get(exchangeRouter.post("/publisher/bounties",authMiddleware(RateLimiterMode.Labs),+ bountyBlocklistMiddleware,+ wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS, { requiresRetrieveFlag: false })),+);++exchangeRouter.put(+ "/publisher/bounties/:id",+ authMiddleware(RateLimiterMode.Labs),+ bountyBlocklistMiddleware,wrap(exchangeProxy(ANALYTICS_TIMEOUT_MS, { requiresRetrieveFlag: false })),);