Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2): extract (#1955)
Gergő Móricz31a81a951bd1
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 7eba40a60..0e1fffc95 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -14,6 +14,7 @@ import { performExtraction_F0 } from "../../lib/extract/fire-0/extraction-servicimport { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { logger as _logger } from "../../lib/logger";+import { fromV1ScrapeOptions } from "../v2/types";export async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,@@ -89,8 +90,15 @@ export async function extractController(zeroDataRetention: req.acuc?.flags?.forceZDR,});+ const scrapeOptions = req.body.scrapeOptions+ ? fromV1ScrapeOptions(req.body.scrapeOptions, req.body.scrapeOptions.timeout, req.auth.team_id).scrapeOptions+ : undefined;+const jobData = {- request: req.body,+ request: {+ ...req.body,+ scrapeOptions,+ },teamId: req.auth.team_id,subId: req.acuc?.sub_id,extractId,diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsnew file mode 100644index 000000000..33e5a7cb4--- /dev/null+++ b/apps/api/src/controllers/v2/extract.ts@@ -0,0 +1,87 @@+import { Response } from "express";+import {+ RequestWithAuth,+ ExtractRequest,+ extractRequestSchema,+ ExtractResponse,+} from "./types";+import { getExtractQueue } from "../../services/queue-service";+import { saveExtract } from "../../lib/extract/extract-redis";+import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";+import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";+import { logger as _logger } from "../../lib/logger";++/**+ * Extracts data from the provided URLs based on the request parameters.+ * Currently in beta.+ * @param req - The request object containing authentication and extraction details.+ * @param res - The response object to send the extraction results.+ * @returns A promise that resolves when the extraction process is complete.+ */+export async function extractController(+ req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,+ res: Response<ExtractResponse>,+) {+ const originalRequest = { ...req.body };+ req.body = extractRequestSchema.parse(req.body);++ if (req.acuc?.flags?.forceZDR) {+ return res.status(400).json({ success: false, error: "Your team has zero data retention enabled. This is not supported on extract. Please contact [email protected] to unblock this feature." });+ }++ const invalidURLs: string[] = req.body.urls?.filter((url: string) => isUrlBlocked(url, req.acuc?.flags ?? null)) ?? [];++ if (invalidURLs.length > 0 && !req.body.ignoreInvalidURLs) {+ if (!res.headersSent) {+ return res.status(403).json({+ success: false,+ error: BLOCKLISTED_URL_MESSAGE,+ });+ }+ }++ const extractId = crypto.randomUUID();++ _logger.info("Extract starting...", {+ request: req.body,+ originalRequest,+ teamId: req.auth.team_id,+ team_id: req.auth.team_id,+ subId: req.acuc?.sub_id,+ extractId,+ zeroDataRetention: req.acuc?.flags?.forceZDR,+ });++ const jobData = {+ request: req.body,+ teamId: req.auth.team_id,+ subId: req.acuc?.sub_id,+ extractId,+ agent: req.body.agent,+ };++ await saveExtract(extractId, {+ id: extractId,+ team_id: req.auth.team_id,+ createdAt: Date.now(),+ status: "processing",+ showSteps: req.body.__experimental_streamSteps,+ showLLMUsage: req.body.__experimental_llmUsage,+ showSources: req.body.__experimental_showSources || req.body.showSources,+ showCostTracking: req.body.__experimental_showCostTracking,+ zeroDataRetention: req.acuc?.flags?.forceZDR,+ });++ await getExtractQueue().add(extractId, jobData, {+ jobId: extractId,+ });++ return res.status(200).json({+ success: true,+ id: extractId,+ urlTrace: [],+ ...(invalidURLs.length > 0 && req.body.ignoreInvalidURLs ? {+ invalidURLs,+ } : {}),+ });+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex ee6c80a30..3f833ccd0 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -9,7 +9,7 @@ import {Document as V0Document,WebSearchResult,} from "../../lib/entities";-import { ScrapeOptions as V1ScrapeOptions } from "../v1/types";+import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";import { InternalOptions } from "../../scraper/scrapeURL";export enum IntegrationEnum {@@ -391,7 +391,7 @@ import type { CostTracking } from "../../lib/extract/extraction-service";const ajv = new Ajv();-export const extractV1Options = z+export const extractOptions = z.object({urls: url.array()@@ -426,6 +426,7 @@ export const extractV1Options = zintegration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),urlTrace: z.boolean().default(false),timeout: z.number().int().positive().finite().safe().optional(),+ agent: agentOptionsExtract.optional(),__experimental_streamSteps: z.boolean().default(false),__experimental_llmUsage: z.boolean().default(false),__experimental_showSources: z.boolean().default(false),@@ -457,8 +458,8 @@ export const extractV1Options = z: x.scrapeOptions,}));-export type ExtractV1Options = z.infer<typeof extractV1Options>;-export const extractRequestSchema = extractV1Options;+export type ExtractOptions = z.infer<typeof extractOptions>;+export const extractRequestSchema = extractOptions;export type ExtractRequest = z.infer<typeof extractRequestSchema>;export type ExtractRequestInput = z.input<typeof extractRequestSchema>;diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex fe2475086..1b84733b2 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -1,13 +1,10 @@-import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v1/types";-import { logger } from "../logger";+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v2/types";import { getScrapeQueue } from "../../services/queue-service";import { waitForJob } from "../../services/queue-jobs";import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";import type { Logger } from "winston";-import { getJobFromGCS } from "../gcs-jobs";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import { fromV1ScrapeOptions } from "../../controllers/v2/types";interface ScrapeDocumentOptions {url: string;@@ -42,19 +39,16 @@ export async function scrapeDocument(from_extract: true,});- const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({- ...internalScrapeOptions,- maxAge: 4 * 60 * 60 * 1000,- }), internalScrapeOptions.timeout, options.teamId)-await addScrapeJob({url: options.url,mode: "single_urls",team_id: options.teamId,- scrapeOptions: scrapeOptions,+ scrapeOptions: scrapeOptionsSchema.parse({+ ...internalScrapeOptions,+ maxAge: 4 * 60 * 60 * 1000,+ }),internalOptions: {- ...internalOptions,teamId: options.teamId,saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,bypassBilling: true,diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 8c2eb3801..65215c60b 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -1,10 +1,10 @@import {Document,ExtractRequest,- isAgentExtractModelValid,TokenUsage,URLTrace,-} from "../../controllers/v1/types";+} from "../../controllers/v2/types";+import { isAgentExtractModelValid } from "../../controllers/v1/types";import { logger as _logger } from "../logger";import { generateBasicCompletion, processUrl } from "./url-processor";import { scrapeDocument } from "./document-scraper";diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex c22828c5a..a4b524a64 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -1,12 +1,10 @@-import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v1/types";-import { logger } from "../../logger";+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v2/types";import { getScrapeQueue } from "../../../services/queue-service";import { waitForJob } from "../../../services/queue-jobs";import { addScrapeJob } from "../../../services/queue-jobs";import { getJobPriority } from "../../job-priority";import type { Logger } from "winston";import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";-import { fromV1ScrapeOptions } from "../../../controllers/v2/types";interface ScrapeDocumentOptions {url: string;@@ -41,10 +39,10 @@ export async function scrapeDocument_F0(from_extract: true,});- const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({+ const scrapeOptions = scrapeOptionsSchema.parse({...internalScrapeOptions,maxAge: 4 * 60 * 60 * 1000,- }), internalScrapeOptions.timeout, options.teamId);+ });await addScrapeJob({@@ -53,7 +51,6 @@ export async function scrapeDocument_F0(team_id: options.teamId,scrapeOptions,internalOptions: {- ...internalOptions,teamId: options.teamId,bypassBilling: true,},diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex f03de5cc4..079489238 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -3,7 +3,7 @@ import {ExtractRequest,TokenUsage,URLTrace,- } from "../../../controllers/v1/types";+ } from "../../../controllers/v2/types";import { logger as _logger } from "../../logger";import { scrapeDocument_F0 } from "./document-scraper-f0";import { billTeam } from "../../../services/billing/credit_billing";