Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2): extract (#1955)

Gergő Móricz31a81a951bd1
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 7eba40a60..0e1fffc95 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -14,6 +14,7 @@ import { performExtraction_F0 } from "../../lib/extract/fire-0/extraction-servic
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { logger as _logger } from "../../lib/logger";
+import { fromV1ScrapeOptions } from "../v2/types";
export async function oldExtract(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
@@ -89,8 +90,15 @@ export async function extractController(
zeroDataRetention: req.acuc?.flags?.forceZDR,
});
+ const scrapeOptions = req.body.scrapeOptions
+ ? fromV1ScrapeOptions(req.body.scrapeOptions, req.body.scrapeOptions.timeout, req.auth.team_id).scrapeOptions
+ : undefined;
+
const jobData = {
- request: req.body,
+ request: {
+ ...req.body,
+ scrapeOptions,
+ },
teamId: req.auth.team_id,
subId: req.acuc?.sub_id,
extractId,
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
new file mode 100644
index 000000000..33e5a7cb4
--- /dev/null
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -0,0 +1,87 @@
+import { Response } from "express";
+import {
+ RequestWithAuth,
+ ExtractRequest,
+ extractRequestSchema,
+ ExtractResponse,
+} from "./types";
+import { getExtractQueue } from "../../services/queue-service";
+import { saveExtract } from "../../lib/extract/extract-redis";
+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { logger as _logger } from "../../lib/logger";
+
+/**
+ * Extracts data from the provided URLs based on the request parameters.
+ * Currently in beta.
+ * @param req - The request object containing authentication and extraction details.
+ * @param res - The response object to send the extraction results.
+ * @returns A promise that resolves when the extraction process is complete.
+ */
+export async function extractController(
+ req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
+ res: Response<ExtractResponse>,
+) {
+ const originalRequest = { ...req.body };
+ req.body = extractRequestSchema.parse(req.body);
+
+ if (req.acuc?.flags?.forceZDR) {
+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on extract. Please contact [email protected] to unblock this feature." });
+ }
+
+ const invalidURLs: string[] = req.body.urls?.filter((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null)) ?? [];
+
+ if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {
+ if (!res.headersSent) {
+ return res.status(403).json({
+ success: false,
+ error: BLOCKLISTED_URL_MESSAGE,
+ });
+ }
+ }
+
+ const extractId = crypto.randomUUID();
+
+ _logger.info("Extract starting...", {
+ request: req.body,
+ originalRequest,
+ teamId: req.auth.team_id,
+ team_id: req.auth.team_id,
+ subId: req.acuc?.sub_id,
+ extractId,
+ zeroDataRetention: req.acuc?.flags?.forceZDR,
+ });
+
+ const jobData = {
+ request: req.body,
+ teamId: req.auth.team_id,
+ subId: req.acuc?.sub_id,
+ extractId,
+ agent: req.body.agent,
+ };
+
+ await saveExtract(extractId, {
+ id: extractId,
+ team_id: req.auth.team_id,
+ createdAt: Date.now(),
+ status: "processing",
+ showSteps: req.body.__experimental_streamSteps,
+ showLLMUsage: req.body.__experimental_llmUsage,
+ showSources: req.body.__experimental_showSources || req.body.showSources,
+ showCostTracking: req.body.__experimental_showCostTracking,
+ zeroDataRetention: req.acuc?.flags?.forceZDR,
+ });
+
+ await getExtractQueue().add(extractId, jobData, {
+ jobId: extractId,
+ });
+
+ return res.status(200).json({
+ success: true,
+ id: extractId,
+ urlTrace: [],
+ ...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs ? {
+ invalidURLs,
+ } : {}),
+ });
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index ee6c80a30..3f833ccd0 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -9,7 +9,7 @@ import {
Document as V0Document,
WebSearchResult,
} from "../../lib/entities";
-import { ScrapeOptions as V1ScrapeOptions } from "../v1/types";
+import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";
import { InternalOptions } from "../../scraper/scrapeURL";
export enum IntegrationEnum {
@@ -391,7 +391,7 @@ import type { CostTracking } from "../../lib/extract/extraction-service";
const ajv = new Ajv();
-export const extractV1Options = z
+export const extractOptions = z
.object({
urls: url
.array()
@@ -426,6 +426,7 @@ export const extractV1Options = z
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
urlTrace: z.boolean().default(false),
timeout: z.number().int().positive().finite().safe().optional(),
+ agent: agentOptionsExtract.optional(),
__experimental_streamSteps: z.boolean().default(false),
__experimental_llmUsage: z.boolean().default(false),
__experimental_showSources: z.boolean().default(false),
@@ -457,8 +458,8 @@ export const extractV1Options = z
: x.scrapeOptions,
}));
-export type ExtractV1Options = z.infer<typeof extractV1Options>;
-export const extractRequestSchema = extractV1Options;
+export type ExtractOptions = z.infer<typeof extractOptions>;
+export const extractRequestSchema = extractOptions;
export type ExtractRequest = z.infer<typeof extractRequestSchema>;
export type ExtractRequestInput = z.input<typeof extractRequestSchema>;
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index fe2475086..1b84733b2 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,13 +1,10 @@
-import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v1/types";
-import { logger } from "../logger";
+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v2/types";
import { getScrapeQueue } from "../../services/queue-service";
import { waitForJob } from "../../services/queue-jobs";
import { addScrapeJob } from "../../services/queue-jobs";
import { getJobPriority } from "../job-priority";
import type { Logger } from "winston";
-import { getJobFromGCS } from "../gcs-jobs";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import { fromV1ScrapeOptions } from "../../controllers/v2/types";
interface ScrapeDocumentOptions {
url: string;
@@ -42,19 +39,16 @@ export async function scrapeDocument(
from_extract: true,
});
- const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({
- ...internalScrapeOptions,
- maxAge: 4 * 60 * 60 * 1000,
- }), internalScrapeOptions.timeout, options.teamId)
-
await addScrapeJob(
{
url: options.url,
mode: "single_urls",
team_id: options.teamId,
- scrapeOptions: scrapeOptions,
+ scrapeOptions: scrapeOptionsSchema.parse({
+ ...internalScrapeOptions,
+ maxAge: 4 * 60 * 60 * 1000,
+ }),
internalOptions: {
- ...internalOptions,
teamId: options.teamId,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
bypassBilling: true,
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index 8c2eb3801..65215c60b 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -1,10 +1,10 @@
import {
Document,
ExtractRequest,
- isAgentExtractModelValid,
TokenUsage,
URLTrace,
-} from "../../controllers/v1/types";
+} from "../../controllers/v2/types";
+import { isAgentExtractModelValid } from "../../controllers/v1/types";
import { logger as _logger } from "../logger";
import { generateBasicCompletion, processUrl } from "./url-processor";
import { scrapeDocument } from "./document-scraper";
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index c22828c5a..a4b524a64 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -1,12 +1,10 @@
-import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v1/types";
-import { logger } from "../../logger";
+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v2/types";
import { getScrapeQueue } from "../../../services/queue-service";
import { waitForJob } from "../../../services/queue-jobs";
import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
-import { fromV1ScrapeOptions } from "../../../controllers/v2/types";
interface ScrapeDocumentOptions {
url: string;
@@ -41,10 +39,10 @@ export async function scrapeDocument_F0(
from_extract: true,
});
- const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({
+ const scrapeOptions = scrapeOptionsSchema.parse({
...internalScrapeOptions,
maxAge: 4 * 60 * 60 * 1000,
- }), internalScrapeOptions.timeout, options.teamId);
+ });
await addScrapeJob(
{
@@ -53,7 +51,6 @@ export async function scrapeDocument_F0(
team_id: options.teamId,
scrapeOptions,
internalOptions: {
- ...internalOptions,
teamId: options.teamId,
bypassBilling: true,
},
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index f03de5cc4..079489238 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -3,7 +3,7 @@ import {
ExtractRequest,
TokenUsage,
URLTrace,
- } from "../../../controllers/v1/types";
+ } from "../../../controllers/v2/types";
import { logger as _logger } from "../../logger";
import { scrapeDocument_F0 } from "./document-scraper-f0";
import { billTeam } from "../../../services/billing/credit_billing";