Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(extract): improve enforcement (#1790)

Gergő Móricze3c45cd4a311
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index b6eecf320..083dd2628 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,4 +1,4 @@
-import { Document, ScrapeOptions, URLTrace, scrapeOptions } from "../../controllers/v1/types";
+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";
import { logger } from "../logger";
import { getScrapeQueue } from "../../services/queue-service";
import { waitForJob } from "../../services/queue-jobs";
@@ -6,6 +6,7 @@ import { addScrapeJob } from "../../services/queue-jobs";
import { getJobPriority } from "../job-priority";
import type { Logger } from "winston";
import { getJobFromGCS } from "../gcs-jobs";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
interface ScrapeDocumentOptions {
url: string;
@@ -13,6 +14,7 @@ interface ScrapeDocumentOptions {
origin: string;
timeout: number;
isSingleUrl?: boolean;
+ flags: TeamFlags | null;
}
export async function scrapeDocument(
@@ -27,6 +29,10 @@ export async function scrapeDocument(
trace.timing.scrapedAt = new Date().toISOString();
}
+ if (isUrlBlocked(options.url, options.flags ?? null)) {
+ return null;
+ }
+
async function attemptScrape(timeout: number) {
const jobId = crypto.randomUUID();
const jobPriority = await getJobPriority({
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index 304cde911..6f60d695d 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -37,6 +37,7 @@ import { normalizeUrl } from "../canonical-url";
import { search } from "../../search";
import { buildRephraseToSerpPrompt } from "./build-prompts";
import { getACUCTeam } from "../../controllers/auth";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
interface ExtractServiceOptions {
request: ExtractRequest;
teamId: string;
@@ -407,6 +408,7 @@ export async function performExtraction(
teamId,
origin: "extract",
timeout,
+ flags: acuc?.flags ?? null,
},
urlTraces,
logger.child({
@@ -742,6 +744,7 @@ export async function performExtraction(
teamId,
origin: "extract",
timeout,
+ flags: acuc?.flags ?? null,
},
urlTraces,
logger.child({
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index 824440b1d..c6ecca291 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -1,10 +1,11 @@
-import { Document, ScrapeOptions, URLTrace, scrapeOptions } from "../../../controllers/v1/types";
+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";
import { logger } from "../../logger";
import { getScrapeQueue } from "../../../services/queue-service";
import { waitForJob } from "../../../services/queue-jobs";
import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
import type { Logger } from "winston";
+import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
interface ScrapeDocumentOptions {
url: string;
@@ -12,6 +13,7 @@ interface ScrapeDocumentOptions {
origin: string;
timeout: number;
isSingleUrl?: boolean;
+ flags: TeamFlags | null;
}
export async function scrapeDocument_F0(
@@ -26,6 +28,10 @@ export async function scrapeDocument_F0(
trace.timing.scrapedAt = new Date().toISOString();
}
+ if (isUrlBlocked(options.url, options.flags ?? null)) {
+ return null;
+ }
+
async function attemptScrape(timeout: number) {
const jobId = crypto.randomUUID();
const jobPriority = await getJobPriority({
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 76b6cdb5f..e35ca6d10 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -344,6 +344,7 @@ import { getACUCTeam } from "../../../controllers/auth";
teamId,
origin: "extract",
timeout,
+ flags: acuc?.flags ?? null,
},
urlTraces,
logger.child({
@@ -624,6 +625,7 @@ import { getACUCTeam } from "../../../controllers/auth";
teamId,
origin: "extract",
timeout,
+ flags: acuc?.flags ?? null,
},
urlTraces,
logger.child({
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index 9f2fb4487..98c079f71 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -147,6 +147,7 @@ export async function performGenerateLlmsTxt(
origin: "llmstxt",
timeout: 30000,
isSingleUrl: true,
+ flags: acuc?.flags ?? null,
},
[],
logger,