Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): content_type in scrapes table

mogery31a6bb598c3d
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 9b6c2c7e0..d22263724 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -426,6 +426,7 @@ export const scrapes = pgTable("scrapes", {
created_at: ts("created_at").notNull().defaultNow(),
monitor_id: uuid("monitor_id"),
monitor_check_id: uuid("monitor_check_id"),
+ content_type: text("content_type"),
});
export const search_feedback = pgTable("search_feedback", {
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index ce29cedc2..7d7ce53bc 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -225,6 +225,7 @@ export type LoggedScrape = {
options: ScrapeOptions;
cost_tracking?: ReturnType<typeof CostTracking.prototype.toJSON>;
pdf_num_pages?: number;
+ content_type?: string | null;
credits_cost: number;
skipNuq: boolean;
zeroDataRetention: boolean;
@@ -273,6 +274,7 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
: {
monitor_id: scrape.monitor_id ?? null,
monitor_check_id: scrape.monitor_check_id ?? null,
+ content_type: scrape.content_type ?? null,
}),
},
force,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 6292f9712..adc009944 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -554,6 +554,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
options: job.data.scrapeOptions,
cost_tracking: costTracking.toJSON(),
pdf_num_pages: doc.metadata.numPages,
+ content_type: doc.metadata.contentType,
credits_cost: credits_billed ?? 0,
zeroDataRetention: job.data.zeroDataRetention,
skipNuq: job.data.skipNuq ?? false,
@@ -642,6 +643,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
options: job.data.scrapeOptions,
cost_tracking: costTracking.toJSON(),
pdf_num_pages: doc.metadata.numPages,
+ content_type: doc.metadata.contentType,
credits_cost: credits_billed ?? 0,
zeroDataRetention: job.data.zeroDataRetention,
skipNuq: job.data.skipNuq ?? false,