Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

v2 scrape changes: change tracking

Gergő Móricz10bfec58f35d
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 37971e9b9..7cbca3392 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -180,7 +180,17 @@ export const jsonFormatWithOptions = z.object({
prompt: z.string().max(10000).optional(),
});
-export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>
+export type JsonFormatWithOptions = z.output<typeof jsonFormatWithOptions>;
+
+export const changeTrackingFormatWithOptions = z.object({
+ type: z.literal("changeTracking"),
+ prompt: z.string().optional(),
+ schema: z.any().optional(),
+ modes: z.enum(["json", "git-diff"]).array().optional().default([]),
+ tag: z.string().or(z.null()).default(null),
+});
+
+export type ChangeTrackingFormatWithOptions = z.output<typeof changeTrackingFormatWithOptions>;
const baseScrapeOptions = z
.object({
@@ -198,6 +208,7 @@ const baseScrapeOptions = z
"changeTracking",
]),
jsonFormatWithOptions,
+ changeTrackingFormatWithOptions,
])
.array()
.optional()
@@ -223,14 +234,6 @@ const baseScrapeOptions = z
.safe()
.max(60000)
.default(0),
- changeTrackingOptions: z
- .object({
- prompt: z.string().optional(),
- schema: z.any().optional(),
- modes: z.enum(["json", "git-diff"]).array().optional().default([]),
- tag: z.string().or(z.null()).default(null),
- })
- .optional(),
mobile: z.boolean().default(false),
parsePDF: z.boolean().default(true),
actions: actionsSchema.optional(),
@@ -282,28 +285,10 @@ const waitForRefineOpts = {
message: "waitFor must not exceed half of timeout",
path: ["waitFor"],
};
-const extractRefine = (obj) => {
- const hasExtractFormat = obj.formats?.includes("extract");
- const hasExtractOptions = obj.extract !== undefined;
- const hasJsonFormat = obj.formats?.includes("json");
- const hasJsonOptions = obj.jsonOptions !== undefined;
- return (
- ((hasExtractFormat && hasExtractOptions) ||
- (!hasExtractFormat && !hasExtractOptions)) &&
- ((hasJsonFormat && hasJsonOptions) || (!hasJsonFormat && !hasJsonOptions))
- );
-};
-const extractRefineOpts = {
- message:
- "When 'extract' or 'json' format is specified, corresponding options must be provided, and vice versa",
-};
const extractTransform = (obj) => {
// Handle timeout
if (
- (obj.formats?.includes("extract") ||
- obj.extract ||
- obj.formats?.includes("json") ||
- obj.jsonOptions) &&
+ (obj.formats.find(x => typeof x === "object" && x.type === "json")) &&
obj.timeout === 30000
) {
obj = { ...obj, timeout: 60000 };
@@ -322,22 +307,6 @@ const extractTransform = (obj) => {
obj = { ...obj, timeout: 120000 };
}
- if (obj.formats?.includes("json")) {
- obj.formats.push("extract");
- }
-
- // Convert JSON options to extract options if needed
- if (obj.jsonOptions && !obj.extract) {
- obj = {
- ...obj,
- extract: {
- prompt: obj.jsonOptions.prompt,
- schema: obj.jsonOptions.schema,
- mode: "llm",
- },
- };
- }
-
return obj;
};
@@ -355,7 +324,6 @@ export const scrapeOptions = baseScrapeOptions
message: `Total wait time (waitFor + wait actions) cannot exceed ${ACTIONS_MAX_WAIT_TIME} seconds`,
},
)
- .refine(extractRefine, extractRefineOpts)
.refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
@@ -423,10 +391,6 @@ export const extractV1Options = z
...obj,
allowExternalLinks: obj.allowExternalLinks || obj.enableWebSearch,
}))
- .refine(
- (x) => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),
- extractRefineOpts,
- )
.refine(
(x) => (x.scrapeOptions ? waitForRefine(x.scrapeOptions) : true),
waitForRefineOpts,
@@ -453,7 +417,6 @@ export const scrapeRequestSchema = baseScrapeOptions
zeroDataRetention: z.boolean().optional(),
})
.strict(strictMessage)
- .refine(extractRefine, extractRefineOpts)
.refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
@@ -492,7 +455,6 @@ export const batchScrapeRequestSchema = baseScrapeOptions
zeroDataRetention: z.boolean().optional(),
})
.strict(strictMessage)
- .refine(extractRefine, extractRefineOpts)
.refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
@@ -508,7 +470,6 @@ export const batchScrapeRequestSchemaNoURLValidation = baseScrapeOptions
zeroDataRetention: z.boolean().optional(),
})
.strict(strictMessage)
- .refine(extractRefine, extractRefineOpts)
.refine(waitForRefine, waitForRefineOpts)
.transform(extractTransform);
@@ -559,7 +520,6 @@ export const crawlRequestSchema = crawlerOptions
zeroDataRetention: z.boolean().optional(),
})
.strict(strictMessage)
- .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
.refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform((x) => {
if (x.crawlEntireDomain !== undefined) {
@@ -1123,6 +1083,16 @@ export function fromV1ScrapeOptions(
return fmt;
} else if (x === "json") {
return null;
+ } else if (x === "changeTracking") {
+ const opts = v1ScrapeOptions.changeTrackingOptions!;
+ const fmt: ChangeTrackingFormatWithOptions = {
+ type: "changeTracking",
+ modes: opts.modes,
+ tag: opts.tag,
+ schema: opts.schema,
+ prompt: opts.prompt,
+ };
+ return fmt;
} else {
return x;
}
@@ -1230,7 +1200,6 @@ export const searchRequestSchema = z
.strict(
"Unrecognized key in body -- please review the v1 API documentation for request body changes",
)
- .refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts)
.refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform((x) => ({
...x,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
index 3086165b8..b46b32986 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts
@@ -7,13 +7,15 @@ import parseDiff from 'parse-diff';
import { generateCompletions } from "./llmExtract";
async function extractDataWithSchema(content: string, meta: Meta): Promise<{ extract: any } | null> {
+ const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")!;
+
try {
const { extract } = await generateCompletions({
logger: meta.logger.child({
method: "extractDataWithSchema/generateCompletions",
}),
options: {
- schema: meta.options.changeTrackingOptions?.schema,
+ schema: changeTrackingFormat.schema,
systemPrompt: "Extract the requested information from the content based on the provided schema.",
temperature: 0
},
@@ -57,7 +59,9 @@ function compareExtractedData(previousData: any, currentData: any): any {
}
export async function deriveDiff(meta: Meta, document: Document): Promise<Document> {
- if (meta.options.formats.includes("changeTracking")) {
+ const changeTrackingFormat = meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking");
+
+ if (changeTrackingFormat) {
if (meta.internalOptions.zeroDataRetention) {
document.warning = "Change tracking is not supported with zero data retention." + (document.warning ? " " + document.warning : "")
return document;
@@ -68,7 +72,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
.rpc("diff_get_last_scrape_4", {
i_team_id: meta.internalOptions.teamId,
i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,
- i_tag: meta.options.changeTrackingOptions?.tag ?? null,
+ i_tag: changeTrackingFormat.tag ?? null,
});
const end = Date.now();
if (end - start > 100) {
@@ -98,7 +102,7 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
visibility: meta.internalOptions.urlInvisibleInCurrentCrawl ? "hidden" : "visible",
}
- if (meta.options.changeTrackingOptions?.modes?.includes("git-diff") && changeStatus === "changed") {
+ if (changeTrackingFormat.modes?.includes("git-diff") && changeStatus === "changed") {
const diffText = gitDiff(previousMarkdown, currentMarkdown, {
color: false,
wordDiff: false
@@ -151,13 +155,12 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
}
}
- if (meta.options.changeTrackingOptions?.modes?.includes("json") &&
- meta.options.changeTrackingOptions && changeStatus === "changed") {
+ if (changeTrackingFormat.modes?.includes("json") && changeStatus === "changed") {
try {
- const previousData = meta.options.changeTrackingOptions.schema ?
+ const previousData = changeTrackingFormat.schema ?
await extractDataWithSchema(previousMarkdown, meta) : null;
- const currentData = meta.options.changeTrackingOptions.schema ?
+ const currentData = changeTrackingFormat.schema ?
await extractDataWithSchema(currentMarkdown, meta) : null;
if (previousData && currentData) {
@@ -169,8 +172,8 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Docume
}),
options: {
systemPrompt: "Analyze the differences between the previous and current content and provide a structured summary of the changes.",
- schema: meta.options.changeTrackingOptions.schema,
- prompt: meta.options.changeTrackingOptions.prompt,
+ schema: changeTrackingFormat.schema,
+ prompt: changeTrackingFormat.prompt,
temperature: 0
},
markdown: `Previous Content:\n${previousMarkdown}\n\nCurrent Content:\n${currentMarkdown}`,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 91299d654..02155b78c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -208,7 +208,7 @@ export function coerceFieldsToFormats(
}
if (document.changeTracking &&
- (!meta.options.changeTrackingOptions?.modes?.includes("git-diff")) &&
+ (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("git-diff")) &&
document.changeTracking.diff !== undefined) {
meta.logger.warn(
"Removed diff from changeTracking because git-diff mode wasn't specified in changeTrackingOptions.modes.",
@@ -217,7 +217,7 @@ export function coerceFieldsToFormats(
}
if (document.changeTracking &&
- (!meta.options.changeTrackingOptions?.modes?.includes("json")) &&
+ (!meta.options.formats.find(x => typeof x === "object" && x.type === "changeTracking")?.modes?.includes("json")) &&
document.changeTracking.json !== undefined) {
meta.logger.warn(
"Removed structured from changeTracking because structured mode wasn't specified in changeTrackingOptions.modes.",