Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

v2 scrape changes + rebase scrapeurl on v2

Gergő Móricz281ce258a3da
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index abd858a91..6abee39fe 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -31,6 +31,7 @@ import { getJobPriority } from "../../lib/job-priority";
import { fromLegacyScrapeOptions, url as urlSchema } from "../v1/types";
import { ZodError } from "zod";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { fromV0ScrapeOptions } from "../v2/types";
export async function crawlController(req: Request, res: Response) {
try {
@@ -158,7 +159,7 @@ export async function crawlController(req: Request, res: Response) {
await logCrawl(id, team_id);
- const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(
+ const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(
pageOptions,
undefined,
undefined,
diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
index cea249335..cc5ab4a2d 100644
--- a/apps/api/src/controllers/v0/crawlPreview.ts
+++ b/apps/api/src/controllers/v0/crawlPreview.ts
@@ -17,6 +17,7 @@ import { checkAndUpdateURL } from "../../../src/lib/validateUrl";
import * as Sentry from "@sentry/node";
import { fromLegacyScrapeOptions } from "../v1/types";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
+import { fromV0ScrapeOptions } from "../v2/types";
export async function crawlPreviewController(req: Request, res: Response) {
try {
@@ -97,7 +98,7 @@ export async function crawlPreviewController(req: Request, res: Response) {
robots = await this.getRobotsTxt();
} catch (_) {}
- const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions(
+ const { scrapeOptions, internalOptions } = fromV0ScrapeOptions(
pageOptions,
undefined,
undefined,
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index a5ec432f1..9d0e005bd 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -32,6 +32,7 @@ import { ZodError } from "zod";
import { Document as V0Document } from "./../../lib/entities";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { fromV0Combo } from "../v2/types";
export async function scrapeHelper(
jobId: string,
@@ -63,7 +64,7 @@ export async function scrapeHelper(
const jobPriority = await getJobPriority({ team_id, basePriority: 10 });
- const { scrapeOptions, internalOptions } = fromLegacyCombo(
+ const { scrapeOptions, internalOptions } = fromV0Combo(
pageOptions,
extractorOptions,
timeout,
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index aaff178c2..a96220883 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -25,6 +25,7 @@ import {
toLegacyDocument,
} from "../v1/types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
+import { fromV0Combo } from "../v2/types";
export async function searchHelper(
jobId: string,
@@ -70,7 +71,7 @@ export async function searchHelper(
let justSearch = pageOptions.fetchPageContent === false;
- const { scrapeOptions, internalOptions } = fromLegacyCombo(
+ const { scrapeOptions, internalOptions } = fromV0Combo(
pageOptions,
undefined,
60000,
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 9a034c150..2f8a3e5ed 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -24,6 +24,7 @@ import { callWebhook } from "../../services/webhook";
import { logger as _logger } from "../../lib/logger";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { fromV1ScrapeOptions } from "../v2/types";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -100,12 +101,15 @@ export async function batchScrapeController(
await logCrawl(id, req.auth.team_id);
}
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body, req.body.timeout, req.auth.team_id);
+
const sc: StoredCrawl = req.body.appendToId
? ((await getCrawl(req.body.appendToId)) as StoredCrawl)
: {
crawlerOptions: null,
- scrapeOptions: req.body,
+ scrapeOptions,
internalOptions: {
+ ...internalOptions,
disableSmartWaitCache: true,
teamId: req.auth.team_id,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
@@ -134,10 +138,6 @@ export async function batchScrapeController(
}
logger.debug("Using job priority " + jobPriority, { jobPriority });
- const scrapeOptions: ScrapeOptions = { ...req.body };
- delete (scrapeOptions as any).urls;
- delete (scrapeOptions as any).appendToId;
-
const jobs = urls.map(x => ({
data: {
url: x,
@@ -152,7 +152,7 @@ export async function batchScrapeController(
v1: true,
webhook: req.body.webhook,
internalOptions: sc.internalOptions,
- zeroDataRetention,
+ zeroDataRetention: zeroDataRetention ?? false,
},
opts: {
jobId: uuidv4(),
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 7c104ccd0..7a0e356e2 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -258,13 +258,13 @@ export async function crawlStatusController(
});
creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (
- sc.scrapeOptions?.extract
+ sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
? 5
: 1
));
} else {
creditsUsed = totalCount * (
- sc.scrapeOptions?.extract
+ sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
? 5
: 1
)
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index d1ae218f1..c482990fb 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -11,6 +11,7 @@ import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
import { logCrawl } from "../../services/logging/crawl_log";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
+import { fromV1ScrapeOptions } from "../v2/types";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -56,7 +57,7 @@ export async function crawlController(
url: undefined,
scrapeOptions: undefined,
};
- const scrapeOptions = req.body.scrapeOptions;
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body.scrapeOptions, req.body.scrapeOptions.timeout, req.auth.team_id);
// TODO: @rafa, is this right? copied from v0
if (Array.isArray(crawlerOptions.includePaths)) {
@@ -92,6 +93,7 @@ export async function crawlController(
crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
scrapeOptions,
internalOptions: {
+ ...internalOptions,
disableSmartWaitCache: true,
teamId: req.auth.team_id,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 6145beced..0405a2571 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -4,10 +4,10 @@ import {
MapDocument,
mapRequestSchema,
RequestWithAuth,
- scrapeOptions,
TeamFlags,
TimeoutSignal,
} from "./types";
+import { scrapeOptions } from "../v2/types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { MapResponse, MapRequest } from "./types";
import { configDotenv } from "dotenv";
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 254bc53a9..19bb7998e 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -11,6 +11,7 @@ import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
import { getScrapeQueue } from "../../services/queue-service";
+import { fromV1ScrapeOptions } from "../v2/types";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -56,19 +57,17 @@ export async function scrapeController(
});
const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(req.body, req.body.timeout, req.auth.team_id);
await addScrapeJob(
{
url: req.body.url,
mode: "single_urls",
team_id: req.auth.team_id,
- scrapeOptions: {
- ...req.body,
- ...(req.body.__experimental_cache ? {
- maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
- } : {}),
- },
+ scrapeOptions,
internalOptions: {
+ ...internalOptions,
teamId: req.auth.team_id,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
unnormalizedSourceURL: preNormalizedBody.url,
@@ -79,7 +78,7 @@ export async function scrapeController(
origin,
integration: req.body.integration,
startTime,
- zeroDataRetention,
+ zeroDataRetention: zeroDataRetention ?? false,
},
{},
jobId,
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 82efbcfb3..c17644f86 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -25,6 +25,7 @@ import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
+import { fromV1ScrapeOptions } from "../v2/types";
interface DocumentWithCostTracking {
document: Document;
@@ -104,16 +105,19 @@ async function scrapeSearchResult(
origin: options.origin,
zeroDataRetention,
});
+
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(options.scrapeOptions, options.timeout, options.teamId);
+
await addScrapeJob(
{
url: searchResult.url,
mode: "single_urls" as Mode,
team_id: options.teamId,
scrapeOptions: {
- ...options.scrapeOptions,
+ ...scrapeOptions,
maxAge: 3 * 24 * 60 * 60 * 1000, // 3 days
},
- internalOptions: { teamId: options.teamId, bypassBilling: true, zeroDataRetention },
+ internalOptions: { ...internalOptions, teamId: options.teamId, bypassBilling: true, zeroDataRetention },
origin: options.origin,
is_scrape: true,
startTime: Date.now(),
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 3cd657c91..2825a7deb 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -446,7 +446,7 @@ const extractRefineOpts = {
message:
"When 'extract' or 'json' format is specified, corresponding options must be provided, and vice versa",
};
-const extractTransform = (obj) => {
+const extractTransform = (obj: ScrapeOptions) => {
// Handle timeout
if (
(obj.formats?.includes("extract") ||
@@ -466,7 +466,7 @@ const extractTransform = (obj) => {
obj = { ...obj, timeout: 60000 };
}
- if (obj.agent) {
+ if ((obj as ScrapeOptions).agent) {
obj = { ...obj, timeout: 300000 };
}
@@ -482,13 +482,7 @@ const extractTransform = (obj) => {
if (obj.jsonOptions && !obj.extract) {
obj = {
...obj,
- extract: {
- prompt: obj.jsonOptions.prompt,
- systemPrompt: obj.jsonOptions.systemPrompt,
- schema: obj.jsonOptions.schema,
- agent: obj.jsonOptions.agent,
- mode: "llm",
- },
+ extract: obj.jsonOptions,
};
}
@@ -500,7 +494,7 @@ export const scrapeOptions = baseScrapeOptions
agent: z
.object({
model: z.string().default(agentExtractModelValue),
- prompt: z.string().optional(),
+ prompt: z.string(),
sessionId: z.string().optional(),
waitBeforeClosingMs: z.number().optional(),
})
@@ -524,7 +518,9 @@ export const scrapeOptions = baseScrapeOptions
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
.refine(waitForRefine, waitForRefineOpts)
- .transform(extractTransform);
+ .transform(obj => {
+ return extractTransform(obj) as typeof obj;
+ });
export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
@@ -631,7 +627,7 @@ export const scrapeRequestSchema = baseScrapeOptions
agent: z
.object({
model: z.string().default(agentExtractModelValue),
- prompt: z.string().optional(),
+ prompt: z.string(),
sessionId: z.string().optional(),
waitBeforeClosingMs: z.number().optional(),
})
@@ -647,7 +643,9 @@ export const scrapeRequestSchema = baseScrapeOptions
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
.refine(waitForRefine, waitForRefineOpts)
- .transform(extractTransform);
+ .transform((obj) => {
+ return extractTransform(obj) as typeof obj;
+ });
export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
export type ScrapeRequestInput = z.input<typeof scrapeRequestSchema>;
@@ -687,7 +685,7 @@ export const batchScrapeRequestSchema = baseScrapeOptions
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
.refine(waitForRefine, waitForRefineOpts)
- .transform(extractTransform);
+ .transform((obj) => extractTransform(obj) as typeof obj);
export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
.extend({
@@ -704,7 +702,7 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
.refine(extractRefine, extractRefineOpts)
.refine(fire1Refine, fire1RefineOpts)
.refine(waitForRefine, waitForRefineOpts)
- .transform(extractTransform);
+ .transform((obj) => extractTransform(obj) as typeof obj);
export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;
export type BatchScrapeRequestInput = z.input<typeof batchScrapeRequestSchema>;
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 94729506a..d52408112 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -258,13 +258,13 @@ export async function crawlStatusController(
});
creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? (totalCount * (
- sc.scrapeOptions?.extract
+ sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
? 5
: 1
));
} else {
creditsUsed = totalCount * (
- sc.scrapeOptions?.extract
+ sc.scrapeOptions.formats.find(x => typeof x === "object" && x.type === "json")
? 5
: 1
)
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 366f28719..77ed423e4 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -5,7 +5,7 @@ import {
crawlRequestSchema,
CrawlResponse,
RequestWithAuth,
- toLegacyCrawlerOptions,
+ toV0CrawlerOptions,
} from "./types";
import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
import { logCrawl } from "../../services/logging/crawl_log";
@@ -89,7 +89,7 @@ export async function crawlController(
const sc: StoredCrawl = {
originUrl: req.body.url,
- crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
+ crawlerOptions: toV0CrawlerOptions(crawlerOptions),
scrapeOptions,
internalOptions: {
disableSmartWaitCache: true,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index daf5371b5..37971e9b9 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -8,9 +8,8 @@ import {
ScrapeActionContent,
Document as V0Document,
} from "../../lib/entities";
+import { ScrapeOptions as V1ScrapeOptions } from "../v1/types";
import { InternalOptions } from "../../scraper/scrapeURL";
-import { getURLDepth } from "../../scraper/WebScraper/utils/maxDepthUtils";
-import { agentExtractModelValue, isAgentExtractModelValid } from "../v1/types";
export enum IntegrationEnum {
DIFY = "dify",
@@ -81,58 +80,7 @@ export const url = z.preprocess(
);
const strictMessage =
- "Unrecognized key in body -- please review the v1 API documentation for request body changes";
-
-
-
-export const extractOptions = z
- .object({
- mode: z.enum(["llm"]).default("llm"),
- schema: z.any().optional(),
- prompt: z.string().max(10000).optional(),
- temperature: z.number().optional(),
- })
- .strict(strictMessage)
- .transform((data) => ({
- ...data,
- systemPrompt: "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required."
- }));
-
-export const extractOptionsWithAgent = z
- .object({
- mode: z.enum(["llm"]).default("llm"),
- schema: z.any().optional(),
- prompt: z.string().max(10000).optional(),
- temperature: z.number().optional(),
- agent: z
- .object({
- model: z.string().default(agentExtractModelValue),
- prompt: z.string().optional(),
- })
- .optional(),
- })
- .strict(strictMessage)
- .transform((data) => ({
- ...data,
- systemPrompt: isAgentExtractModelValid(data.agent?.model)
- ? `You are an expert web data extractor. Your task is to analyze the provided markdown content from a web page and generate a JSON object based *strictly* on the provided schema.
-
-Key Instructions:
-1. **Schema Adherence:** Populate the JSON object according to the structure defined in the schema.
-2. **Content Grounding:** Extract information *only* if it is explicitly present in the provided markdown. Do NOT infer or fabricate information.
-3. **Missing Information:** If a piece of information required by the schema cannot be found in the markdown, use \`null\` for that field's value.
-4. **SmartScrape Recommendation:**
- * Assess if the *full* required data seems unavailable in the current markdown likely because:
- - Content requires user interaction to reveal (e.g., clicking buttons, hovering, scrolling)
- - Content uses pagination (e.g., "Load More" buttons, numbered pagination, infinite scroll)
- - Content is dynamically loaded after user actions
- * If the content requires user interaction or pagination to be fully accessible, set \`shouldUseSmartscrape\` to \`true\` in your response and provide a clear \`reasoning\` and \`prompt\` for the SmartScrape tool.
- * If the content is simply JavaScript rendered but doesn't require interaction, set \`shouldUseSmartscrape\` to \`false\`.
-5. **Output Format:** Your final output MUST be a single, valid JSON object conforming precisely to the schema. Do not include any explanatory text outside the JSON structure.`
- : "Based on the information on the page, extract all the information from the schema in JSON format. Try to extract all the fields even those that might not be marked as required."
- }));
-
-export type ExtractOptions = z.infer<typeof extractOptions>;
+ "Unrecognized key in body -- please review the v2 API documentation for request body changes";
const ACTIONS_MAX_WAIT_TIME = 60;
const MAX_ACTIONS = 50;
@@ -226,20 +174,30 @@ export const actionsSchema = z
},
);
+export const jsonFormatWithOptions = z.object({
+ type: z.literal("json"),
+ schema: z.any().optional(),
+ prompt: z.string().max(10000).optional(),
+});
+
+export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>
+
const baseScrapeOptions = z
.object({
formats: z
- .enum([
- "markdown",
- "html",
- "rawHtml",
- "links",
- "screenshot",
- "screenshot@fullPage",
- "extract",
- "json",
- "summary",
- "changeTracking",
+ .union([
+ z.enum([
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "screenshot",
+ "screenshot@fullPage",
+ "extract",
+ "summary",
+ "changeTracking",
+ ]),
+ jsonFormatWithOptions,
])
.array()
.optional()
@@ -265,10 +223,6 @@ const baseScrapeOptions = z
.safe()
.max(60000)
.default(0),
- // Deprecate this to jsonOptions
- extract: extractOptions.optional(),
- // New
- jsonOptions: extractOptions.optional(),
changeTrackingOptions: z
.object({
prompt: z.string().optional(),
@@ -280,7 +234,7 @@ const baseScrapeOptions = z
mobile: z.boolean().default(false),
parsePDF: z.boolean().default(true),
actions: actionsSchema.optional(),
- // New
+
location: z
.object({
country: z
@@ -301,23 +255,6 @@ const baseScrapeOptions = z
})
.optional(),
- // Deprecated
- geolocation: z
- .object({
- country: z
- .string()
- .optional()
- .refine(
- (val) => !val || Object.keys(countries).includes(val.toUpperCase()),
- {
- message:
- "Invalid country code. Please use a valid ISO 3166-1 alpha-2 country code.",
- },
- )
- .transform((val) => (val ? val.toUpperCase() : "US-generic")),
- languages: z.string().array().optional(),
- })
- .optional(),
skipTlsVerification: z.boolean().default(true),
removeBase64Images: z.boolean().default(true),
fastMode: z.boolean().default(false),
@@ -327,7 +264,6 @@ const baseScrapeOptions = z
maxAge: z.number().int().gte(0).safe().default(0),
storeInCache: z.boolean().default(true),
// @deprecated
- __experimental_cache: z.boolean().default(false).optional(),
__searchPreviewToken: z.string().optional(),
__experimental_omce: z.boolean().default(false).optional(),
})
@@ -406,10 +342,6 @@ const extractTransform = (obj) => {
};
export const scrapeOptions = baseScrapeOptions
- .extend({
- extract: extractOptions.optional(),
- jsonOptions: extractOptions.optional(),
- })
.strict(strictMessage)
.refine(
(obj) => {
@@ -429,10 +361,7 @@ export const scrapeOptions = baseScrapeOptions
export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
-export type ScrapeOptions = BaseScrapeOptions & {
- extract?: z.infer<typeof extractOptions>,
- jsonOptions?: z.infer<typeof extractOptions>,
-};
+export type ScrapeOptions = BaseScrapeOptions;
import Ajv from "ajv";
import type { CostTracking } from "../../lib/extract/extraction-service";
@@ -518,8 +447,6 @@ export const scrapeRequestSchema = baseScrapeOptions
.omit({ timeout: true })
.extend({
url,
- extract: extractOptions.optional(),
- jsonOptions: extractOptions.optional(),
origin: z.string().optional().default("api"),
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
timeout: z.number().int().positive().finite().safe().default(30000),
@@ -1027,7 +954,7 @@ export interface ResponseWithSentry<ResBody = undefined>
sentry?: string;
}
-export function toLegacyCrawlerOptions(x: CrawlerOptions) {
+export function toV0CrawlerOptions(x: CrawlerOptions) {
return {
includes: x.includePaths,
excludes: x.excludePaths,
@@ -1049,7 +976,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {
};
}
-export function toNewCrawlerOptions(x: any): CrawlerOptions {
+export function toV2CrawlerOptions(x: any): CrawlerOptions {
return {
includePaths: x.includes,
excludePaths: x.excludes,
@@ -1068,7 +995,7 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {
}
}
-export function fromLegacyCrawlerOptions(x: any, teamId: string): {
+export function fromV0CrawlerOptions(x: any, teamId: string): {
crawlOptions: CrawlerOptions;
internalOptions: InternalOptions;
} {
@@ -1101,7 +1028,7 @@ export interface MapDocument {
title?: string;
description?: string;
}
-export function fromLegacyScrapeOptions(
+export function fromV0ScrapeOptions(
pageOptions: PageOptions,
extractorOptions: ExtractorOptions | undefined,
timeout: number | undefined,
@@ -1119,7 +1046,11 @@ export function fromLegacyScrapeOptions(
: null,
extractorOptions !== undefined &&
extractorOptions.mode.includes("llm-extraction")
- ? ("extract" as const)
+ ? ({
+ type: "json" as const,
+ prompt: extractorOptions.userPrompt,
+ schema: extractorOptions.extractionSchema,
+ })
: null,
"links",
].filter((x) => x !== null),
@@ -1140,15 +1071,6 @@ export function fromLegacyScrapeOptions(
location: pageOptions.geolocation,
skipTlsVerification: pageOptions.skipTlsVerification,
removeBase64Images: pageOptions.removeBase64Images,
- extract:
- extractorOptions !== undefined &&
- extractorOptions.mode.includes("llm-extraction")
- ? {
- systemPrompt: extractorOptions.extractionPrompt,
- prompt: extractorOptions.userPrompt,
- schema: extractorOptions.extractionSchema,
- }
- : undefined,
mobile: pageOptions.mobile,
fastMode: pageOptions.useFastMode,
}),
@@ -1156,25 +1078,79 @@ export function fromLegacyScrapeOptions(
atsv: pageOptions.atsv,
v0DisableJsDom: pageOptions.disableJsDom,
teamId,
+ ...(extractorOptions !== undefined && extractorOptions.mode.includes("llm-extraction") ? {
+ v1JSONSystemPrompt: extractorOptions.extractionPrompt
+ } : {})
},
// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
};
}
-export function fromLegacyCombo(
+export function fromV1ScrapeOptions(
+ v1ScrapeOptions: V1ScrapeOptions,
+ timeout: number | undefined,
+ teamId: string,
+): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
+ const spreadScrapeOptions = { ...v1ScrapeOptions };
+ delete (spreadScrapeOptions as any).urls;
+ delete (spreadScrapeOptions as any).ignoreInvalidURLs;
+ delete (spreadScrapeOptions as any).url;
+ delete (spreadScrapeOptions as any).origin;
+ delete (spreadScrapeOptions as any).integration;
+
+ delete spreadScrapeOptions.__experimental_cache;
+ delete spreadScrapeOptions.jsonOptions;
+ delete spreadScrapeOptions.extract;
+ delete spreadScrapeOptions.geolocation;
+
+ return {
+ scrapeOptions: scrapeOptions.parse({
+ ...spreadScrapeOptions,
+
+ ...(v1ScrapeOptions.__experimental_cache ? {
+ maxAge: v1ScrapeOptions.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
+ } : {}),
+ location: v1ScrapeOptions.location ?? v1ScrapeOptions.geolocation,
+ formats: v1ScrapeOptions.formats.map(x => {
+ // json and extract is standardized down to extract fmt in v1 -- fine to take one and dismiss the other
+ if (x === "extract") {
+ const opts = v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract;
+ const fmt: JsonFormatWithOptions = {
+ type: "json",
+ schema: opts?.schema,
+ prompt: opts?.prompt,
+ };
+ return fmt;
+ } else if (x === "json") {
+ return null;
+ } else {
+ return x;
+ }
+ }).filter(x => x !== null),
+ }),
+ internalOptions: {
+ teamId,
+ v1Agent: v1ScrapeOptions.agent,
+ v1JSONSystemPrompt: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.systemPrompt,
+ v1JSONAgent: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract)?.agent,
+ },
+ };
+}
+
+export function fromV0Combo(
pageOptions: PageOptions,
extractorOptions: ExtractorOptions | undefined,
timeout: number | undefined,
crawlerOptions: any,
teamId: string,
): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {
- const { scrapeOptions, internalOptions: i1 } = fromLegacyScrapeOptions(
+ const { scrapeOptions, internalOptions: i1 } = fromV0ScrapeOptions(
pageOptions,
extractorOptions,
timeout,
teamId,
);
- const { internalOptions: i2 } = fromLegacyCrawlerOptions(crawlerOptions, teamId);
+ const { internalOptions: i2 } = fromV0CrawlerOptions(crawlerOptions, teamId);
return { scrapeOptions, internalOptions: Object.assign(i1, i2) };
}
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 2bf7f12bd..d68b4d400 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -1,5 +1,5 @@
import { InternalOptions } from "../scraper/scrapeURL";
-import { ScrapeOptions, TeamFlags } from "../controllers/v1/types";
+import { ScrapeOptions, TeamFlags } from "../controllers/v2/types";
import { WebCrawler } from "../scraper/WebScraper/crawler";
import { redisEvictConnection } from "../services/redis";
import { logger as _logger } from "./logger";
diff --git a/apps/api/src/lib/deep-research/research-manager.ts b/apps/api/src/lib/deep-research/research-manager.ts
index 0f9618a99..a70c4b6c7 100644
--- a/apps/api/src/lib/deep-research/research-manager.ts
+++ b/apps/api/src/lib/deep-research/research-manager.ts
@@ -160,7 +160,6 @@ export class ResearchLLMService {
method: "generateSearchQueries",
}),
options: {
- mode: "llm",
systemPrompt:
"You are an expert research agent that generates search queries (SERP) to explore topics deeply and thoroughly. Do not generate repeated queries. Today's date is " +
new Date().toISOString().split("T")[0],
@@ -224,7 +223,6 @@ export class ResearchLLMService {
method: "analyzeAndPlan",
}),
options: {
- mode: "llm",
systemPrompt:
systemPrompt +
"You are an expert research agent that is analyzing findings. Your goal is to synthesize information and identify gaps for further research. Today's date is " +
diff --git a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts
index 922ebeb97..99c5181d8 100644
--- a/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts
+++ b/apps/api/src/lib/extract/completions/analyzeSchemaAndPrompt.ts
@@ -50,7 +50,6 @@ export async function analyzeSchemaAndPrompt(
const { extract: result, totalUsage } = await generateCompletions({
logger,
options: {
- mode: "llm",
schema: checkSchema,
prompt: buildAnalyzeSchemaUserPrompt(schemaString, prompt, urls),
systemPrompt: buildAnalyzeSchemaPrompt(),
diff --git a/apps/api/src/lib/extract/completions/batchExtract.ts b/apps/api/src/lib/extract/completions/batchExtract.ts
index e6c71211e..3a7602007 100644
--- a/apps/api/src/lib/extract/completions/batchExtract.ts
+++ b/apps/api/src/lib/extract/completions/batchExtract.ts
@@ -63,7 +63,6 @@ export async function batchExtractPromise(options: BatchExtractOptions, logger:
method: "extractService/generateCompletions",
}),
options: {
- mode: "llm",
systemPrompt: buildBatchExtractSystemPrompt(
systemPrompt,
multiEntitySchema,
diff --git a/apps/api/src/lib/extract/completions/checkShouldExtract.ts b/apps/api/src/lib/extract/completions/checkShouldExtract.ts
index 6f678ee83..3f8ad859e 100644
--- a/apps/api/src/lib/extract/completions/checkShouldExtract.ts
+++ b/apps/api/src/lib/extract/completions/checkShouldExtract.ts
@@ -18,7 +18,6 @@ export async function checkShouldExtract(
const shouldExtractCheck = await generateCompletions({
logger: logger.child({ method: "extractService/checkShouldExtract" }),
options: {
- mode: "llm",
systemPrompt: buildShouldExtractSystemPrompt(),
prompt: buildShouldExtractUserPrompt(prompt, multiEntitySchema),
schema: {
diff --git a/apps/api/src/lib/extract/completions/singleAnswer.ts b/apps/api/src/lib/extract/completions/singleAnswer.ts
index 7260c1c48..f3355f7b6 100644
--- a/apps/api/src/lib/extract/completions/singleAnswer.ts
+++ b/apps/api/src/lib/extract/completions/singleAnswer.ts
@@ -42,7 +42,6 @@ export async function singleAnswerCompletion({
extractId,
}),
options: {
- mode: "llm",
systemPrompt:
(systemPrompt ? `${systemPrompt}\n` : "") +
"Always prioritize using the provided content to answer the question. Do not make up an answer. Do not hallucinate. In case you can't find the information and the string is required, instead of 'N/A' or 'Not speficied', return an empty string: '', if it's not a string and you can't find the information, return null. Be concise and follow the schema always if provided.",
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 083dd2628..fe2475086 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,4 +1,4 @@
-import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";
+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../controllers/v1/types";
import { logger } from "../logger";
import { getScrapeQueue } from "../../services/queue-service";
import { waitForJob } from "../../services/queue-jobs";
@@ -7,6 +7,7 @@ import { getJobPriority } from "../job-priority";
import type { Logger } from "winston";
import { getJobFromGCS } from "../gcs-jobs";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { fromV1ScrapeOptions } from "../../controllers/v2/types";
interface ScrapeDocumentOptions {
url: string;
@@ -41,16 +42,19 @@ export async function scrapeDocument(
from_extract: true,
});
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({
+ ...internalScrapeOptions,
+ maxAge: 4 * 60 * 60 * 1000,
+ }), internalScrapeOptions.timeout, options.teamId)
+
await addScrapeJob(
{
url: options.url,
mode: "single_urls",
team_id: options.teamId,
- scrapeOptions: scrapeOptions.parse({
- ...internalScrapeOptions,
- maxAge: 4 * 60 * 60 * 1000,
- }),
+ scrapeOptions: scrapeOptions,
internalOptions: {
+ ...internalOptions,
teamId: options.teamId,
saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
bypassBilling: true,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index c6ecca291..c22828c5a 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -1,4 +1,4 @@
-import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";
+import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions as scrapeOptionsSchema } from "../../../controllers/v1/types";
import { logger } from "../../logger";
import { getScrapeQueue } from "../../../services/queue-service";
import { waitForJob } from "../../../services/queue-jobs";
@@ -6,6 +6,7 @@ import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
import type { Logger } from "winston";
import { isUrlBlocked } from "../../../scraper/WebScraper/utils/blocklist";
+import { fromV1ScrapeOptions } from "../../../controllers/v2/types";
interface ScrapeDocumentOptions {
url: string;
@@ -40,16 +41,19 @@ export async function scrapeDocument_F0(
from_extract: true,
});
+ const { scrapeOptions, internalOptions } = fromV1ScrapeOptions(scrapeOptionsSchema.parse({
+ ...internalScrapeOptions,
+ maxAge: 4 * 60 * 60 * 1000,
+ }), internalScrapeOptions.timeout, options.teamId);
+
await addScrapeJob(
{
url: options.url,
mode: "single_urls",
team_id: options.teamId,
- scrapeOptions: scrapeOptions.parse({
- ...internalScrapeOptions,
- maxAge: 4 * 60 * 60 * 1000,
- }),
+ scrapeOptions,
internalOptions: {
+ ...internalOptions,
teamId: options.teamId,
bypassBilling: true,
},
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
index efe82d4bc..1814acf60 100644
--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
@@ -336,32 +336,6 @@ export async function generateCompletions_F0({
}
}
-export async function performLLMExtract(
- meta: Meta,
- document: Document,
-): Promise<Document> {
- if (meta.options.formats.includes("extract")) {
- meta.internalOptions.abort?.throwIfAborted();
- const { extract, warning } = await generateCompletions_F0({
- logger: meta.logger.child({
- method: "performLLMExtract/generateCompletions",
- }),
- options: meta.options.extract!,
- markdown: document.markdown,
- previousWarning: document.warning
- });
-
- if (meta.options.formats.includes("json")) {
- document.json = extract;
- } else {
- document.extract = extract;
- }
- document.warning = warning;
- }
-
- return document;
-}
-
export function removeDefaultProperty_F0(schema: any): any {
if (typeof schema !== "object" || schema === null) return schema;
diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
index 16ad1840c..033c3408d 100644
--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
@@ -229,7 +229,6 @@ export async function rerankLinksWithLLM_F0(options: RerankerOptions, costTracki
retry,
}),
options: {
- mode: "llm",
systemPrompt: buildRerankerSystemPrompt_F0(),
prompt: buildRerankerUserPrompt_F0(searchQuery),
schema: schema,
diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts
index 370924691..1e56b7ef8 100644
--- a/apps/api/src/lib/extract/reranker.ts
+++ b/apps/api/src/lib/extract/reranker.ts
@@ -306,7 +306,6 @@ export async function rerankLinksWithLLM(
retry,
}),
options: {
- mode: "llm",
systemPrompt: systemPrompt,
prompt: buildRerankerUserPrompt(searchQuery),
schema: schema,
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index 98c079f71..2504e0bbc 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -168,7 +168,6 @@ export async function performGenerateLlmsTxt(
model: getModel("gpt-4o-mini", "openai"),
options: {
systemPrompt: "",
- mode: "llm",
schema: descriptionSchema,
prompt: `Generate a 9-10 word description and a 3-4 word title of the entire page based on ALL the content one will find on the page for this url: ${document.metadata?.url}. This will help in a user finding the page for its intended purpose.`,
},
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 30981f8a1..34e7adcd2 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -1,7 +1,7 @@
import { parseStringPromise } from "xml2js";
import { WebCrawler } from "./crawler";
import { scrapeURL } from "../scrapeURL";
-import { scrapeOptions, TimeoutSignal } from "../../controllers/v1/types";
+import { scrapeOptions, TimeoutSignal } from "../../controllers/v2/types";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { parseSitemapXml, processSitemap } from "../../lib/crawler";
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 65459bc02..93fc3ddc9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -239,7 +239,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
}
: {}),
priority: meta.internalOptions.priority,
- geolocation: meta.options.geolocation ?? meta.options.location,
+ geolocation: meta.options.location,
mobile: meta.options.mobile,
timeout, // TODO: better timeout logic
disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
@@ -356,7 +356,7 @@ export async function scrapeURLWithFireEnginePlaywright(
screenshot: meta.options.formats.includes("screenshot"),
fullPageScreenshot: meta.options.formats.includes("screenshot@fullPage"),
wait: meta.options.waitFor,
- geolocation: meta.options.geolocation ?? meta.options.location,
+ geolocation: meta.options.location,
blockAds: meta.options.blockAds,
mobileProxy: meta.featureFlags.has("stealthProxy"),
@@ -421,7 +421,7 @@ export async function scrapeURLWithFireEngineTLSClient(
priority: meta.internalOptions.priority,
atsv: meta.internalOptions.atsv,
- geolocation: meta.options.geolocation ?? meta.options.location,
+ geolocation: meta.options.location,
disableJsDom: meta.internalOptions.v0DisableJsDom,
mobileProxy: meta.featureFlags.has("stealthProxy"),
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index f6e2d3749..b4608587f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,8 +1,9 @@
import { Logger } from "winston";
import * as Sentry from "@sentry/node";
-import { Document, ScrapeOptions, TimeoutSignal, TeamFlags } from "../../controllers/v1/types";
-import { logger as _logger, logger } from "../../lib/logger";
+import { Document, ScrapeOptions, TimeoutSignal, TeamFlags } from "../../controllers/v2/types";
+import { ScrapeOptions as ScrapeOptionsV1 } from "../../controllers/v1/types";
+import { logger as _logger } from "../../lib/logger";
import {
buildFallbackList,
Engine,
@@ -99,7 +100,7 @@ function buildFeatureFlags(
flags.add("atsv");
}
- if (options.location || options.geolocation) {
+ if (options.location) {
flags.add("location");
}
@@ -222,6 +223,10 @@ export type InternalOptions = {
bypassBilling?: boolean;
zeroDataRetention?: boolean;
teamFlags?: TeamFlags;
+
+ v1Agent?: ScrapeOptionsV1["agent"];
+ v1JSONAgent?: Exclude<ScrapeOptionsV1["jsonOptions"], undefined>["agent"];
+ v1JSONSystemPrompt?: string;
};
export type EngineResultsTracker = {
@@ -293,7 +298,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const timeToRun =
meta.options.timeout !== undefined
? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))
- : (!meta.options.actions && !meta.options.jsonOptions && !meta.options.extract)
+ : (!meta.options.actions && !meta.options.formats.find(x => typeof x === "object" && x.type === "json"))
? Math.round(120000 / Math.min(fallbackList.length, 2))
: undefined;
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
index 25dfb6ef4..c71013984 100644
--- a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -1,7 +1,7 @@
// TODO: refactor
import { AnyNode, Cheerio, load } from "cheerio"; // rustified
-import { ScrapeOptions } from "../../../controllers/v1/types";
+import { ScrapeOptions } from "../../../controllers/v2/types";
import { transformHtml } from "../../../lib/html-transformer";
import { logger } from "../../../lib/logger";
import { queryOMCESignatures } from "../../../services/index";
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index 531cf1a28..a9a6188f3 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -3,7 +3,7 @@ import "dotenv/config";
process.env.ENV = "test";
import { scrapeURL } from ".";
-import { scrapeOptions } from "../../controllers/v1/types";
+import { scrapeOptions } from "../../controllers/v2/types";
import { Engine } from "./engines";
import { CostTracking } from "../../lib/extract/extraction-service";
diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.ts
index 0b8474056..207d1c480 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts
@@ -11,7 +11,7 @@ export async function performAgent(
meta: Meta,
document: Document,
): Promise<Document> {
- if (meta.options.agent?.prompt) {
+ if (meta.internalOptions.v1Agent?.prompt) {
if (meta.internalOptions.zeroDataRetention) {
document.warning = "Agent is not supported with zero data retention." + (document.warning ? " " + document.warning : "")
return document;
@@ -25,8 +25,8 @@ export async function performAgent(
return document;
}
- const prompt = meta.options.agent?.prompt ?? undefined
- const sessionId = meta.options.agent?.sessionId ?? undefined
+ const prompt = meta.internalOptions.v1Agent?.prompt ?? undefined
+ const sessionId = meta.internalOptions.v1Agent?.sessionId ?? undefined
let smartscrapeResults: SmartScrapeResult;
try {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
index 79a54b745..3086165b8 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
@@ -13,7 +13,6 @@ async function extractDataWithSchema(content: string, meta: Meta): Promise<{ ext
method: "extractDataWithSchema/generateCompletions",
}),
options: {
- mode: "llm",
schema: meta.options.changeTrackingOptions?.schema,
systemPrompt: "Extract the requested information from the content based on the provided schema.",
temperature: 0
@@ -169,7 +168,6 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
method: "deriveDiff/generateCompletions",
}),
options: {
- mode: "llm",
systemPrompt: "Analyze the differences between the previous and current content and provide a structured summary of the changes.",
schema: meta.options.changeTrackingOptions.schema,
prompt: meta.options.changeTrackingOptions.prompt,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 77e564a49..45d083918 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -2,10 +2,9 @@ import { encoding_for_model } from "@dqbd/tiktoken";
import { TiktokenModel } from "@dqbd/tiktoken";
import {
Document,
- ExtractOptions,
- isAgentExtractModelValid,
+ JsonFormatWithOptions,
TokenUsage,
-} from "../../../controllers/v1/types";
+} from "../../../controllers/v2/types";
import { Logger } from "winston";
import { EngineResultsTracker, Meta } from "..";
import { logger } from "../../../lib/logger";
@@ -24,6 +23,7 @@ import fs from "fs/promises";
import Ajv from "ajv";
import { extractData } from "../lib/extractSmartScrape";
import { CostTracking } from "../../../lib/extract/extraction-service";
+import { isAgentExtractModelValid } from "../../../controllers/v1/types";
// TODO: fix this, it's horrible
type LanguageModelV1ProviderMetadata = {
anthropic?: {
@@ -225,7 +225,10 @@ export function calculateCost(
export type GenerateCompletionsOptions = {
model?: LanguageModel;
logger: Logger;
- options: ExtractOptions;
+ options: Omit<JsonFormatWithOptions, "type"> & {
+ systemPrompt?: string;
+ temperature?: number;
+ };
markdown?: string;
previousWarning?: string;
isExtractEndpoint?: boolean;
@@ -653,7 +656,8 @@ export async function performLLMExtract(
meta: Meta,
document: Document,
): Promise<Document> {
- if (meta.options.formats.includes("extract")) {
+ const jsonFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "json") as JsonFormatWithOptions | undefined;
+ if (jsonFormat) {
if (meta.internalOptions.zeroDataRetention) {
document.warning = "JSON mode is not supported with zero data retention." + (document.warning ? " " + document.warning : "")
return document;
@@ -667,7 +671,7 @@ export async function performLLMExtract(
logger: meta.logger.child({
method: "performLLMExtract/generateCompletions",
}),
- options: meta.options.extract!,
+ options: jsonFormat,
markdown: document.markdown,
previousWarning: document.warning,
// ... existing model and provider options ...
@@ -691,7 +695,7 @@ export async function performLLMExtract(
await extractData({
extractOptions: generationOptions,
urls: [meta.rewrittenUrl ?? meta.url],
- useAgent: false,
+ useAgent: isAgentExtractModelValid(meta.internalOptions.v1JSONAgent?.model),
scrapeId: meta.id,
});
@@ -775,11 +779,7 @@ export async function performLLMExtract(
// }
// Assign the final extracted data
- if (meta.options.formats.includes("json")) {
- document.json = extractedData;
- } else {
- document.extract = extractedData;
- }
+ document.json = extractedData;
// document.warning = warning;
}
@@ -801,7 +801,6 @@ export async function performSummary(
method: "performSummary/generateCompletions",
}),
options: {
- mode: "llm",
systemPrompt: "You are a content summarization expert. Analyze the provided content and create a concise, informative summary that captures the key points, main ideas, and essential information. Focus on clarity and brevity while maintaining accuracy.",
prompt: "Summarize the main content and key points from this page.",
},
@@ -908,7 +907,6 @@ export async function generateSchemaFromPrompt(
retryModel,
markdown: "",
options: {
- mode: "llm",
systemPrompt: `You are a schema generator for a web scraping system. Generate a JSON schema based on the user's prompt.
Consider:
1. The type of data being requested
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 2ce073f64..9c829baa4 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -13,7 +13,7 @@ import systemMonitor from "../system-monitor";
import { v4 as uuidv4 } from "uuid";
import { index_supabase_service, processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs } from "..";
import { processWebhookInsertJobs } from "../webhook";
-import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toLegacyCrawlerOptions } from "../../controllers/v1/types";
+import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toV0CrawlerOptions } from "../../controllers/v2/types";
import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../queue-jobs";
@@ -133,7 +133,7 @@ const processPrecrawlJobInternal = async (token: string, job: Job) => {
const sc: StoredCrawl = {
originUrl: url,
- crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),
+ crawlerOptions: toV0CrawlerOptions(crawlerOptions),
scrapeOptions,
internalOptions: {
disableSmartWaitCache: true,
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index ae692f203..2c82afec6 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -45,7 +45,7 @@ import {
} from "../../src/lib/job-priority";
import { getJobs } from "..//controllers/v1/crawl-status";
import { configDotenv } from "dotenv";
-import { scrapeOptions } from "../controllers/v1/types";
+import { scrapeOptions } from "../controllers/v2/types";
import {
concurrentJobDone,
pushConcurrencyLimitActiveJob,
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index fb21013e3..4f43ff104 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -3,10 +3,10 @@ import {
AuthCreditUsageChunk,
BaseScrapeOptions,
ScrapeOptions,
- Document as V1Document,
+ Document as V2Document,
webhookSchema,
TeamFlags,
-} from "./controllers/v1/types";
+} from "./controllers/v2/types";
import { ExtractorOptions, Document } from "./lib/entities";
import { InternalOptions } from "./scraper/scrapeURL";
import type { CostTracking } from "./lib/extract/extraction-service";
@@ -85,7 +85,7 @@ export type RunWebScraperResult =
}
| {
success: true;
- document: V1Document;
+ document: V2Document;
};
export interface FirecrawlJob {