Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
compare format (FIR-1560) (#1405)
Gergő Móricz24f519935916
diff --git a/apps/api/requests.http b/apps/api/requests.httpindex a39973714..4c69d011c 100644--- a/apps/api/requests.http+++ b/apps/api/requests.http@@ -19,7 +19,7 @@ Authorization: Bearer {{$dotenv TEST_API_KEY}}content-type: application/json{- "url": "https://firecrawl.dev"+ "url":"https://firecrawl.dev"}### Check Crawl Statusdiff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 2b697ca97..7495e7891 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -84,6 +84,18 @@ describe("Scrape tests", () => {// expect(response.markdown).toMatch(/(\.g\.doubleclick\.net|amazon-adsystem\.com)\//);// }, 30000);// });++ describe("Compare format", () => {+ it.concurrent("works", async () => {+ const response = await scrape({+ url: "https://example.com",+ formats: ["markdown", "compare"],+ });++ expect(response.compare).toBeDefined();+ expect(response.compare?.previousScrapeAt).not.toBeNull();+ });+ });describe("Location API (f-e dependant)", () => {it.concurrent("works without specifying an explicit location", async () => {diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 2eba651d0..c8b186b06 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -158,6 +158,7 @@ export async function crawlController(req: Request, res: Response) {pageOptions,undefined,undefined,+ team_id);internalOptions.disableSmartWaitCache = true; // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterdiff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.tsindex ffb8ebba7..9153ea79c 100644--- a/apps/api/src/controllers/v0/crawlPreview.ts+++ b/apps/api/src/controllers/v0/crawlPreview.ts@@ -99,6 +99,7 @@ export async function crawlPreviewController(req: Request, res: Response) {pageOptions,undefined,undefined,+ team_id);const sc: StoredCrawl = {diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 62d62b092..0bdd197bc 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -66,6 +66,7 @@ export async function scrapeHelper(extractorOptions,timeout,crawlerOptions,+ team_id,);await addScrapeJob(@@ -297,6 +298,7 @@ export async function scrapeController(req: Request, res: Response) {pageOptions,extractorOptions,timeout,+ team_id,);logJob({diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex ac7d7f621..d8649a52c 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -72,6 +72,7 @@ export async function searchHelper(undefined,60000,crawlerOptions,+ team_id,);if (justSearch) {diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex d2c079bf4..20fab47c3 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -82,7 +82,7 @@ export async function batchScrapeController(: {crawlerOptions: null,scrapeOptions: req.body,- internalOptions: { disableSmartWaitCache: true }, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matter+ internalOptions: { disableSmartWaitCache: true, teamId: req.auth.team_id }, // NOTE: smart wait disabled for batch scrapes to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),plan: req.auth.plan,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 51d373eed..31e39502f 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -81,7 +81,7 @@ export async function crawlController(originUrl: req.body.url,crawlerOptions: toLegacyCrawlerOptions(crawlerOptions),scrapeOptions,- internalOptions: { disableSmartWaitCache: true }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter+ internalOptions: { disableSmartWaitCache: true, teamId: req.auth.team_id }, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteam_id: req.auth.team_id,createdAt: Date.now(),plan: req.auth.plan,diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex ebb0b3246..49890d90b 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -85,7 +85,7 @@ export async function getMapResults({scrapeOptions: undefined,},scrapeOptions: scrapeOptions.parse({}),- internalOptions: {},+ internalOptions: { teamId },team_id: teamId,createdAt: Date.now(),plan: plan,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex ec11e2cb9..44214ee2e 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -50,7 +50,7 @@ export async function scrapeController(mode: "single_urls",team_id: req.auth.team_id,scrapeOptions: req.body,- internalOptions: {},+ internalOptions: { teamId: req.auth.team_id },plan: req.auth.plan!,origin: req.body.origin,is_scrape: true,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 18ff9579c..082cd8cdf 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -83,7 +83,7 @@ async function scrapeSearchResult(mode: "single_urls" as Mode,team_id: options.teamId,scrapeOptions: options.scrapeOptions,- internalOptions: {},+ internalOptions: { teamId: options.teamId },plan: options.plan || "free",origin: options.origin,is_scrape: true,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 459e5e560..b610826d3 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -20,7 +20,8 @@ export type Format =| "links"| "screenshot"| "screenshot@fullPage"- | "extract";+ | "extract"+ | "compare";export const url = z.preprocess((x) => {@@ -165,6 +166,7 @@ const baseScrapeOptions = z"screenshot@fullPage","extract","json",+ "compare",]).array().optional()@@ -172,6 +174,10 @@ const baseScrapeOptions = z.refine((x) => !(x.includes("screenshot") && x.includes("screenshot@fullPage")),"You may only specify either screenshot or screenshot@fullPage",+ )+ .refine(+ (x) => !x.includes("compare") || x.includes("markdown"),+ "The compare format requires the markdown format to be specified as well",),headers: z.record(z.string(), z.string()).optional(),includeTags: z.string().array().optional(),@@ -546,6 +552,11 @@ export type Document = {value: unknown}[];};+ compare?: {+ previousScrapeAt: string | null;+ changeStatus: "new" | "same" | "changed" | "removed";+ visibility: "visible" | "hidden";+ }metadata: {title?: string;description?: string;@@ -812,7 +823,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {};}-export function fromLegacyCrawlerOptions(x: any): {+export function fromLegacyCrawlerOptions(x: any, teamId: string): {crawlOptions: CrawlerOptions;internalOptions: InternalOptions;} {@@ -834,6 +845,7 @@ export function fromLegacyCrawlerOptions(x: any): {}),internalOptions: {v0CrawlOnlyUrls: x.returnOnlyUrls,+ teamId,},};}@@ -847,6 +859,7 @@ export function fromLegacyScrapeOptions(pageOptions: PageOptions,extractorOptions: ExtractorOptions | undefined,timeout: number | undefined,+ teamId: string,): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {return {scrapeOptions: scrapeOptions.parse({@@ -896,6 +909,7 @@ export function fromLegacyScrapeOptions(internalOptions: {atsv: pageOptions.atsv,v0DisableJsDom: pageOptions.disableJsDom,+ teamId,},// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks};@@ -906,13 +920,15 @@ export function fromLegacyCombo(extractorOptions: ExtractorOptions | undefined,timeout: number | undefined,crawlerOptions: any,+ teamId: string,): { scrapeOptions: ScrapeOptions; internalOptions: InternalOptions } {const { scrapeOptions, internalOptions: i1 } = fromLegacyScrapeOptions(pageOptions,extractorOptions,timeout,+ teamId,);- const { internalOptions: i2 } = fromLegacyCrawlerOptions(crawlerOptions);+ const { internalOptions: i2 } = fromLegacyCrawlerOptions(crawlerOptions, teamId);return { scrapeOptions, internalOptions: Object.assign(i1, i2) };}diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex b741e6151..e261800fb 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -154,20 +154,20 @@ export async function finishCrawlKickoff(id: string) {);}-export async function finishCrawl(id: string) {+export async function finishCrawlPre(id: string) {if (await isCrawlFinished(id)) {- _logger.debug("Marking crawl as finished.", {+ _logger.debug("Marking crawl as pre-finished.", {module: "crawl-redis",- method: "finishCrawl",+ method: "finishCrawlPre",crawlId: id,});- const set = await redisConnection.setnx("crawl:" + id + ":finish", "yes");- await redisConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);+ const set = await redisConnection.setnx("crawl:" + id + ":finished_pre", "yes");+ await redisConnection.expire("crawl:" + id + ":finished_pre", 24 * 60 * 60);return set === 1;} else {- _logger.debug("Crawl can not be finished yet, not marking as finished.", {+ _logger.debug("Crawl can not be pre-finished yet, not marking as finished.", {module: "crawl-redis",- method: "finishCrawl",+ method: "finishCrawlPre",crawlId: id,jobs_done: await redisConnection.scard("crawl:" + id + ":jobs_done"),jobs: await redisConnection.scard("crawl:" + id + ":jobs"),@@ -177,6 +177,16 @@ export async function finishCrawl(id: string) {}}+export async function finishCrawl(id: string) {+ _logger.debug("Marking crawl as finished.", {+ module: "crawl-redis",+ method: "finishCrawl",+ crawlId: id,+ });+ await redisConnection.set("crawl:" + id + ":finish", "yes");+ await redisConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);+}+export async function getCrawlJobs(id: string): Promise<string[]> {return await redisConnection.smembers("crawl:" + id + ":jobs");}@@ -250,7 +260,7 @@ export function generateURLPermutations(url: string | URL): URL[] {return [urlWithHTML, urlWithPHP, urlWithSlash, urlWithBare];});- return permutations;+ return [...new Set(permutations.map(x => x.href))].map(x => new URL(x));}export async function lockURL(diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 8cbc75fde..e9bd729aa 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -44,6 +44,7 @@ export async function scrapeDocument(scrapeOptions: scrapeOptions.parse({ ...internalScrapeOptions }),internalOptions: {useCache: true,+ teamId: options.teamId,},plan: options.plan,origin: options.origin,diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex c67512180..ba983e07b 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -51,6 +51,7 @@ export async function startWebScraperPipeline({priority: job.opts.priority,is_scrape: job.data.is_scrape ?? false,is_crawl: !!(job.data.crawl_id && job.data.crawlerOptions !== null),+ urlInvisibleInCurrentCrawl: job.data.crawlerOptions?.urlInvisibleInCurrentCrawl ?? false,});}@@ -66,6 +67,7 @@ export async function runWebScraper({priority,is_scrape = false,is_crawl = false,+ urlInvisibleInCurrentCrawl = false,}: RunWebScraperParams): Promise<ScrapeUrlResponse> {const logger = _logger.child({method: "runWebScraper",@@ -97,6 +99,8 @@ export async function runWebScraper({response = await scrapeURL(bull_job_id, url, scrapeOptions, {priority,...internalOptions,+ urlInvisibleInCurrentCrawl,+ teamId: internalOptions?.teamId ?? team_id,});if (!response.success) {if (response.error instanceof Error) {diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex c2c603834..f945cd220 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -47,6 +47,7 @@ export async function getLinksFromSitemap(],v0DisableJsDom: true,abort,+ teamId: "sitemap",},);diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex eaf5497af..e047d1cb6 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -162,6 +162,8 @@ async function buildMetaObject(}export type InternalOptions = {+ teamId: string;+priority?: number; // Passed along to fire-engineforceEngine?: Engine | Engine[];atsv?: boolean; // anti-bot solver, beta@@ -173,6 +175,7 @@ export type InternalOptions = {isBackgroundIndex?: boolean;fromCache?: boolean; // Indicates if the document was retrieved from cacheabort?: AbortSignal;+ urlInvisibleInCurrentCrawl?: boolean;};export type EngineResultsTracker = {@@ -383,7 +386,7 @@ export async function scrapeURL(id: string,url: string,options: ScrapeOptions,- internalOptions: InternalOptions = {},+ internalOptions: InternalOptions,): Promise<ScrapeUrlResponse> {const meta = await buildMetaObject(id, url, options, internalOptions);try {diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex 8b7838211..b545266f9 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -31,7 +31,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-basic","https://www.roastmywebsite.ai/",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -78,7 +78,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["markdown", "html"],}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -102,7 +102,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({onlyMainContent: false,}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -125,7 +125,7 @@ describe("Standalone scrapeURL tests", () => {onlyMainContent: false,excludeTags: [".nav", "#footer", "strong"],}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -145,7 +145,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-400","https://httpstat.us/400",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -163,7 +163,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-401","https://httpstat.us/401",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -181,7 +181,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-403","https://httpstat.us/403",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -199,7 +199,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-404","https://httpstat.us/404",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -217,7 +217,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-405","https://httpstat.us/405",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -235,7 +235,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-500","https://httpstat.us/500",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -253,7 +253,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-redirect","https://scrapethissite.com/",scrapeOptions.parse({}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -285,7 +285,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["screenshot"],}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -313,7 +313,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["screenshot@fullPage"],}),- { forceEngine },+ { forceEngine, teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -341,6 +341,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-pdf","https://arxiv.org/pdf/astro-ph/9301001.pdf",scrapeOptions.parse({}),+ { teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -359,6 +360,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-docx","https://nvca.org/wp-content/uploads/2019/06/NVCA-Model-Document-Stock-Purchase-Agreement.docx",scrapeOptions.parse({}),+ { teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -395,6 +397,7 @@ describe("Standalone scrapeURL tests", () => {},},}),+ { teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -430,6 +433,7 @@ describe("Standalone scrapeURL tests", () => {},},}),+ { teamId: "test" },);// expect(out.logs.length).toBeGreaterThan(0);@@ -451,7 +455,7 @@ describe("Standalone scrapeURL tests", () => {async (i) => {const url = "https://www.scrapethissite.com/?i=" + i;const id = "test:concurrent:" + url;- const out = await scrapeURL(id, url, scrapeOptions.parse({}));+ const out = await scrapeURL(id, url, scrapeOptions.parse({}), { teamId: "test" });const replacer = (key: string, value: any) => {if (value instanceof Error) {diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsnew file mode 100644index 000000000..9628844db--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -0,0 +1,42 @@+import { supabase_service } from "../../../services/supabase";+import { Document } from "../../../controllers/v1/types";+import { Meta } from "../index";++export async function deriveDiff(meta: Meta, document: Document): Promise<Document> {+ if (meta.options.formats.includes("compare")) {+ const res = await supabase_service+ .rpc("diff_get_last_scrape_1", {+ i_team_id: meta.internalOptions.teamId,+ i_url: document.metadata.sourceURL ?? meta.url,+ });++ const data: {+ o_docs: Document[],+ o_date_added: string,+ } | undefined | null = (res.data ?? [])[0] as any;++ if (data && data.o_docs.length > 0) {+ const previousMarkdown = data.o_docs[0].markdown!;+ const currentMarkdown = document.markdown!;++ const transformer = (x: string) => [...x.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")].sort().join("");++ document.compare = {+ previousScrapeAt: data.o_date_added,+ changeStatus: document.metadata.statusCode === 404 ? "removed" : transformer(previousMarkdown) === transformer(currentMarkdown) ? "same" : "changed",+ visibility: meta.internalOptions.urlInvisibleInCurrentCrawl ? "hidden" : "visible",+ }+ } else if (!res.error) {+ document.compare = {+ previousScrapeAt: null,+ changeStatus: document.metadata.statusCode === 404 ? "removed" : "new",+ visibility: meta.internalOptions.urlInvisibleInCurrentCrawl ? "hidden" : "visible",+ }+ } else {+ meta.logger.error("Error fetching previous scrape", { error: res.error });+ document.warning = "Comparing failed, please try again later." + (document.warning ? ` ${document.warning}` : "");+ }+ }++ return document;+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex ea149dba7..114c59a89 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -8,7 +8,7 @@ import { performLLMExtract } from "./llmExtract";import { uploadScreenshot } from "./uploadScreenshot";import { removeBase64Images } from "./removeBase64Images";import { saveToCache } from "./cache";-+import { deriveDiff } from "./diff";export type Transformer = (meta: Meta,document: Document,@@ -148,6 +148,17 @@ export function coerceFieldsToFormats();}+ if (!formats.has("compare") && document.compare !== undefined) {+ meta.logger.warn(+ "Removed compare from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",+ );+ delete document.compare;+ } else if (formats.has("compare") && document.compare === undefined) {+ meta.logger.warn(+ "Request had format compare, but there was no compare field in the result.",+ );+ }+if (meta.options.actions === undefined || meta.options.actions.length === 0) {delete document.actions;}@@ -164,6 +175,7 @@ export const transformerStack: Transformer[] = [deriveMetadataFromRawHTML,uploadScreenshot,performLLMExtract,+ deriveDiff,coerceFieldsToFormats,removeBase64Images,];diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex b6cfc4543..a7da10f77 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -28,6 +28,7 @@ import {addCrawlJobs,crawlToCrawler,finishCrawl,+ finishCrawlPre,finishCrawlKickoff,generateURLPermutations,getCrawl,@@ -100,7 +101,77 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;const runningJobs: Set<string> = new Set();async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {- if (await finishCrawl(job.data.crawl_id)) {+ if (await finishCrawlPre(job.data.crawl_id)) {+ if (job.data.crawlerOptions && !await redisConnection.exists("crawl:" + job.data.crawl_id + ":invisible_urls")) {+ await redisConnection.set("crawl:" + job.data.crawl_id + ":invisible_urls", "done", "EX", 60 * 60 * 24);++ const sc = (await getCrawl(job.data.crawl_id))!;++ const visitedUrls = new Set(await redisConnection.smembers(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ ));++ const lastUrls: string[] = ((await supabase_service.rpc("diff_get_last_crawl_urls", {+ i_team_id: job.data.team_id,+ i_url: sc.originUrl!,+ })).data ?? []).map(x => x.url);++ const lastUrlsSet = new Set(lastUrls);++ const univistedUrls = Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x));+ const addableJobCount = sc.crawlerOptions.limit === undefined ? Infinity : (sc.crawlerOptions.limit - await getDoneJobsOrderedLength(job.data.crawl_id));++ console.log(sc.originUrl!, univistedUrls, visitedUrls, lastUrls, addableJobCount);++ if (univistedUrls.length !== 0 && addableJobCount > 0) {+ const jobs = univistedUrls.slice(0, addableJobCount).map((url) => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ plan: job.data.plan!,+ crawlerOptions: {+ ...job.data.crawlerOptions,+ urlInvisibleInCurrentCrawl: true,+ },+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ const lockedIds = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map((x) => ({ id: x.opts.jobId, url: x.data.url })),+ );+ const lockedJobs = jobs.filter((x) =>+ lockedIds.find((y) => y.id === x.opts.jobId),+ );+ await addCrawlJobs(+ job.data.crawl_id,+ lockedJobs.map((x) => x.opts.jobId),+ );+ await addScrapeJobs(lockedJobs);++ return;+ }+ }++ await finishCrawl(job.data.crawl_id);+(async () => {const originUrl = sc.originUrl? normalizeUrlOnlyHostname(sc.originUrl)diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex f051b1834..adc7df374 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -59,6 +59,7 @@ export interface RunWebScraperParams {priority?: number;is_scrape?: boolean;is_crawl?: boolean;+ urlInvisibleInCurrentCrawl?: boolean;}export type RunWebScraperResult =diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 0e6dbf7c4..772f2a3eb 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "1.21.0",+ "version": "1.21.1","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 41d13da0d..bec5288fb 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -69,6 +69,11 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResultscreenshot?: string;metadata?: FirecrawlDocumentMetadata;actions: ActionsSchema;+ compare?: {+ previousScrapeAt: string | null;+ changeStatus: "new" | "same" | "changed" | "removed";+ visibility: "visible" | "hidden";+ };// v1 search onlytitle?: string;description?: string;@@ -79,7 +84,7 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult* Defines the options and configurations available for scraping web content.*/export interface CrawlScrapeOptions {- formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];+ formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];headers?: Record<string, string>;includeTags?: string[];excludeTags?: string[];