Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
v2 scrape changes: change tracking
Gergő Móricz10bfec58f35d
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 37971e9b9..7cbca3392 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -180,7 +180,17 @@ export const jsonFormatWithOptions = z.object({prompt: z.string().max(10000).optional(),});-export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>+export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;++export const changeTrackingFormatWithOptions = z.object({+ type: z.literal("changeTracking"),+ prompt: z.string().optional(),+ schema: z.any().optional(),+ modes: z.enum(["json", "git-diff"]).array().optional().default([]),+ tag: z.string().or(z.null()).default(null),+});++export type ChangeTrackingFormatWithOptions = z.output<typeof changeTrackingFormatWithOptions>;const baseScrapeOptions = z.object({@@ -198,6 +208,7 @@ const baseScrapeOptions = z"changeTracking",]),jsonFormatWithOptions,+ changeTrackingFormatWithOptions,]).array().optional()@@ -223,14 +234,6 @@ const baseScrapeOptions = z.safe().max(60000).default(0),- changeTrackingOptions: z- .object({- prompt: z.string().optional(),- schema: z.any().optional(),- modes: z.enum(["json", "git-diff"]).array().optional().default([]),- tag: z.string().or(z.null()).default(null),- })- .optional(),mobile: z.boolean().default(false),parsePDF: z.boolean().default(true),actions: actionsSchema.optional(),@@ -282,28 +285,10 @@ const waitForRefineOpts = {message: "waitFor must not exceed half of timeout",path: ["waitFor"],};-const extractRefine = (obj) => {- const hasExtractFormat = obj.formats?.includes("extract");- const hasExtractOptions = obj.extract !== undefined;- const hasJsonFormat = obj.formats?.includes("json");- const hasJsonOptions = obj.jsonOptions !== undefined;- return (- ((hasExtractFormat && hasExtractOptions) ||- (!hasExtractFormat && !hasExtractOptions)) &&- ((hasJsonFormat && hasJsonOptions) || (!hasJsonFormat && !hasJsonOptions))- );-};-const extractRefineOpts = {- message:- "When 'extract' or 'json' format is specified, corresponding options must be provided, and vice versa",-};const extractTransform = (obj) => {// Handle timeoutif (- (obj.formats?.includes("extract") ||- obj.extract ||- obj.formats?.includes("json") ||- obj.jsonOptions) &&+ (obj.formats.find(x => typeof x === "object" && x.type === "json")) &&obj.timeout === 30000) {obj = { ...obj, timeout: 60000 };@@ -322,22 +307,6 @@ const extractTransform = (obj) => {obj = { ...obj, timeout: 120000 };}- if (obj.formats?.includes("json")) {- obj.formats.push("extract");- }-- // Convert JSON options to extract options if needed- if (obj.jsonOptions && !obj.extract) {- obj = {- ...obj,- extract: {- prompt: obj.jsonOptions.prompt,- schema: obj.jsonOptions.schema,- mode: "llm",- },- };- }-return obj;};@@ -355,7 +324,6 @@ export const scrapeOptions = baseScrapeOptionsmessage: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,},)- .refine(extractRefine, extractRefineOpts).refine(waitForRefine, waitForRefineOpts).transform(extractTransform);@@ -423,10 +391,6 @@ export const extractV1Options = z...obj,allowExternalLinks: obj.allowExternalLinks || obj.enableWebSearch,}))- .refine(- (x) => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),- extractRefineOpts,- ).refine((x) => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),waitForRefineOpts,@@ -453,7 +417,6 @@ export const scrapeRequestSchema = baseScrapeOptionszeroDataRetention: z.boolean().optional(),}).strict(strictMessage)- .refine(extractRefine, extractRefineOpts).refine(waitForRefine, waitForRefineOpts).transform(extractTransform);@@ -492,7 +455,6 @@ export const batchScrapeRequestSchema = baseScrapeOptionszeroDataRetention: z.boolean().optional(),}).strict(strictMessage)- .refine(extractRefine, extractRefineOpts).refine(waitForRefine, waitForRefineOpts).transform(extractTransform);@@ -508,7 +470,6 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptionszeroDataRetention: z.boolean().optional(),}).strict(strictMessage)- .refine(extractRefine, extractRefineOpts).refine(waitForRefine, waitForRefineOpts).transform(extractTransform);@@ -559,7 +520,6 @@ export const crawlRequestSchema = crawlerOptionszeroDataRetention: z.boolean().optional(),}).strict(strictMessage)- .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts).refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform((x) => {if (x.crawlEntireDomain !== undefined) {@@ -1123,6 +1083,16 @@ export function fromV1ScrapeOptions(return fmt;} else if (x === "json") {return null;+ } else if (x === "changeTracking") {+ const opts = v1ScrapeOptions.changeTrackingOptions!;+ const fmt: ChangeTrackingFormatWithOptions = {+ type: "changeTracking",+ modes: opts.modes,+ tag: opts.tag,+ schema: opts.schema,+ prompt: opts.prompt,+ };+ return fmt;} else {return x;}@@ -1230,7 +1200,6 @@ export const searchRequestSchema = z.strict("Unrecognized key in body -- please review the v1 API documentation for request body changes",)- .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts).refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform((x) => ({...x,diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 3086165b8..b46b32986 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -7,13 +7,15 @@ import parseDiff from 'parse-diff';import { generateCompletions } from "./llmExtract";async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {+ const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")!;+try {const { extract } = await generateCompletions({logger: meta.logger.child({method: "extractDataWithSchema/generateCompletions",}),options: {- schema: meta.options.changeTrackingOptions?.schema,+ schema: changeTrackingFormat.schema,systemPrompt: "Extract the requested information from the content based on the provided schema.",temperature: 0},@@ -57,7 +59,9 @@ function compareExtractedData(previousData: any, currentData: any): any {}export async function deriveDiff(meta: Meta, document: Document): Promise<Document> {- if (meta.options.formats.includes("changeTracking")) {+ const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking");++ if (changeTrackingFormat) {if (meta.internalOptions.zeroDataRetention) {document.warning = "Change tracking is not supported with zero data retention." + (document.warning ? " " + document.warning : "")return document;@@ -68,7 +72,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume.rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,- i_tag: meta.options.changeTrackingOptions?.tag ?? null,+ i_tag: changeTrackingFormat.tag ?? null,});const end = Date.now();if (end - start > 100) {@@ -98,7 +102,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documevisibility: meta.internalOptions.urlInvisibleInCurrentCrawl ? "hidden" : "visible",}- if (meta.options.changeTrackingOptions?.modes?.includes("git-diff") && changeStatus === "changed") {+ if (changeTrackingFormat.modes?.includes("git-diff") && changeStatus === "changed") {const diffText = gitDiff(previousMarkdown, currentMarkdown, {color: false,wordDiff: false@@ -151,13 +155,12 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}}- if (meta.options.changeTrackingOptions?.modes?.includes("json") &&- meta.options.changeTrackingOptions && changeStatus === "changed") {+ if (changeTrackingFormat.modes?.includes("json") && changeStatus === "changed") {try {- const previousData = meta.options.changeTrackingOptions.schema ?+ const previousData = changeTrackingFormat.schema ?await extractDataWithSchema(previousMarkdown, meta) : null;- const currentData = meta.options.changeTrackingOptions.schema ?+ const currentData = changeTrackingFormat.schema ?await extractDataWithSchema(currentMarkdown, meta) : null;if (previousData && currentData) {@@ -169,8 +172,8 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume}),options: {systemPrompt: "Analyze the differences between the previous and current content and provide a structured summary of the changes.",- schema: meta.options.changeTrackingOptions.schema,- prompt: meta.options.changeTrackingOptions.prompt,+ schema: changeTrackingFormat.schema,+ prompt: changeTrackingFormat.prompt,temperature: 0},markdown: `Previous Content:\n${previousMarkdown}\n\nCurrent Content:\n${currentMarkdown}`,diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 91299d654..02155b78c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -208,7 +208,7 @@ export function coerceFieldsToFormats(}if (document.changeTracking &&- (!meta.options.changeTrackingOptions?.modes?.includes("git-diff")) &&+ (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("git-diff")) &&document.changeTracking.diff !== undefined) {meta.logger.warn("Removed diff from changeTracking because git-diff mode wasn't specified in changeTrackingOptions.modes.",@@ -217,7 +217,7 @@ export function coerceFieldsToFormats(}if (document.changeTracking &&- (!meta.options.changeTrackingOptions?.modes?.includes("json")) &&+ (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("json")) &&document.changeTracking.json !== undefined) {meta.logger.warn("Removed structured from changeTracking because structured mode wasn't specified in changeTrackingOptions.modes.",