Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): webhook data format regression
amplitudes18888f9a42b0
diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.tsindex 39e33689b..e431ff84b 100644--- a/apps/api/src/services/webhook/types.ts+++ b/apps/api/src/services/webhook/types.ts@@ -1,6 +1,7 @@import { z } from "zod";import { webhookSchema } from "./schema";import { ExtractResult } from "../../lib/extract/extraction-service";+import { Document } from "../../controllers/v2/types";export enum WebhookEvent {CRAWL_STARTED = "crawl.started",@@ -52,13 +53,13 @@ export interface CrawlStartedData extends BaseWebhookData {export interface CrawlPageData extends BaseWebhookData {success: boolean;- data: WebhookDocument[] | WebhookDocumentLink[]; // links or documents (v0 compatible)+ data: Document[] | WebhookDocumentLink[]; // links or documents (v0 compatible)error?: string;}export interface CrawlCompletedData extends BaseWebhookData {success: true;- data: WebhookDocument[] | WebhookDocumentLink[]; // empty array or links (v0 compatible)+ data: Document[] | WebhookDocumentLink[]; // empty array or links (v0 compatible)}export interface CrawlFailedData extends BaseWebhookData {@@ -73,7 +74,7 @@ export interface BatchScrapeStartedData extends BaseWebhookData {export interface BatchScrapePageData extends BaseWebhookData {success: boolean;- data: WebhookDocument[];+ data: Document[];error?: string; // more v0 tomfoolery}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 69fd6aa7b..070856e31 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -460,11 +460,7 @@ async function processJob(job: Job & { id: string }) {});if (sender) {const documents = Array.isArray(data?.result?.links)- ? data.result.links.map((x: any) => ({- content: x?.content?.content,- markdown: x?.content?.markdown,- metadata: x?.content?.metadata,- }))+ ? data.result.links.map(x => x.content): [];if (job.data.crawlerOptions !== null) {sender.send(WebhookEvent.CRAWL_PAGE, {