Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(extract): improve enforcement (#1790)
Gergő Móricze3c45cd4a311
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex b6eecf320..083dd2628 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -1,4 +1,4 @@-import { Document, ScrapeOptions, URLTrace, scrapeOptions } from "../../controllers/v1/types";+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";import { logger } from "../logger";import { getScrapeQueue } from "../../services/queue-service";import { waitForJob } from "../../services/queue-jobs";@@ -6,6 +6,7 @@ import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";import type { Logger } from "winston";import { getJobFromGCS } from "../gcs-jobs";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";interface ScrapeDocumentOptions {url: string;@@ -13,6 +14,7 @@ interface ScrapeDocumentOptions {origin: string;timeout: number;isSingleUrl?: boolean;+ flags: TeamFlags | null;}export async function scrapeDocument(@@ -27,6 +29,10 @@ export async function scrapeDocument(trace.timing.scrapedAt = new Date().toISOString();}+ if (isUrlBlocked(options.url, options.flags ?? null)) {+ return null;+ }+async function attemptScrape(timeout: number) {const jobId = crypto.randomUUID();const jobPriority = await getJobPriority({diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 304cde911..6f60d695d 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -37,6 +37,7 @@ import { normalizeUrl } from "../canonical-url";import { search } from "../../search";import { buildRephraseToSerpPrompt } from "./build-prompts";import { getACUCTeam } from "../../controllers/auth";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";interface ExtractServiceOptions {request: ExtractRequest;teamId: string;@@ -407,6 +408,7 @@ export async function performExtraction(teamId,origin: "extract",timeout,+ flags: acuc?.flags ?? null,},urlTraces,logger.child({@@ -742,6 +744,7 @@ export async function performExtraction(teamId,origin: "extract",timeout,+ flags: acuc?.flags ?? null,},urlTraces,logger.child({diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex 824440b1d..c6ecca291 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -1,10 +1,11 @@-import { Document, ScrapeOptions, URLTrace, scrapeOptions } from "../../../controllers/v1/types";+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";import { logger } from "../../logger";import { getScrapeQueue } from "../../../services/queue-service";import { waitForJob } from "../../../services/queue-jobs";import { addScrapeJob } from "../../../services/queue-jobs";import { getJobPriority } from "../../job-priority";import type { Logger } from "winston";+import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";interface ScrapeDocumentOptions {url: string;@@ -12,6 +13,7 @@ interface ScrapeDocumentOptions {origin: string;timeout: number;isSingleUrl?: boolean;+ flags: TeamFlags | null;}export async function scrapeDocument_F0(@@ -26,6 +28,10 @@ export async function scrapeDocument_F0(trace.timing.scrapedAt = new Date().toISOString();}+ if (isUrlBlocked(options.url, options.flags ?? null)) {+ return null;+ }+async function attemptScrape(timeout: number) {const jobId = crypto.randomUUID();const jobPriority = await getJobPriority({diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 76b6cdb5f..e35ca6d10 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -344,6 +344,7 @@ import { getACUCTeam } from "../../../controllers/auth";teamId,origin: "extract",timeout,+ flags: acuc?.flags ?? null,},urlTraces,logger.child({@@ -624,6 +625,7 @@ import { getACUCTeam } from "../../../controllers/auth";teamId,origin: "extract",timeout,+ flags: acuc?.flags ?? null,},urlTraces,logger.child({diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex 9f2fb4487..98c079f71 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -147,6 +147,7 @@ export async function performGenerateLlmsTxt(origin: "llmstxt",timeout: 30000,isSingleUrl: true,+ flags: acuc?.flags ?? null,},[],logger,