Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): content_type in scrapes table
mogery31a6bb598c3d
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 9b6c2c7e0..d22263724 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -426,6 +426,7 @@ export const scrapes = pgTable("scrapes", {created_at: ts("created_at").notNull().defaultNow(),monitor_id: uuid("monitor_id"),monitor_check_id: uuid("monitor_check_id"),+ content_type: text("content_type"),});export const search_feedback = pgTable("search_feedback", {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex ce29cedc2..7d7ce53bc 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -225,6 +225,7 @@ export type LoggedScrape = {options: ScrapeOptions;cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;pdf_num_pages?: number;+ content_type?: string | null;credits_cost: number;skipNuq: boolean;zeroDataRetention: boolean;@@ -273,6 +274,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {: {monitor_id: scrape.monitor_id ?? null,monitor_check_id: scrape.monitor_check_id ?? null,+ content_type: scrape.content_type ?? null,}),},force,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 6292f9712..adc009944 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -554,6 +554,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {options: job.data.scrapeOptions,cost_tracking: costTracking.toJSON(),pdf_num_pages: doc.metadata.numPages,+ content_type: doc.metadata.contentType,credits_cost: credits_billed ?? 0,zeroDataRetention: job.data.zeroDataRetention,skipNuq: job.data.skipNuq ?? false,@@ -642,6 +643,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {options: job.data.scrapeOptions,cost_tracking: costTracking.toJSON(),pdf_num_pages: doc.metadata.numPages,+ content_type: doc.metadata.contentType,credits_cost: credits_billed ?? 0,zeroDataRetention: job.data.zeroDataRetention,skipNuq: job.data.skipNuq ?? false,