Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

merge main: re-port monitor billing onto drizzle + deterministicjson

Developers Digest281c9717e7fd
diff --cc apps/api/src/services/monitoring/results.ts
index 2641ef741,7ba63ef99..1dd663822
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@@ -194,12 -194,9 +195,13 @@@ export async function recordMonitorScra
diff_text_bytes: diffTextBytes,
diff_json_bytes: diffJsonBytes,
status_code: getDocumentStatusCode(doc),
+ ...(error ? { error } : {}),
metadata: {
title: doc?.metadata?.title ?? null,
+ contentType: doc?.metadata?.contentType ?? null,
+ numPages: doc?.metadata?.numPages ?? null,
+ proxyUsed: doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,
creditsUsed: doc?.metadata?.creditsUsed ?? null,
},
judgment: judgment ?? null,
diff --cc apps/api/src/services/monitoring/runner.ts
index ae26cf328,58354e9fa..5b73f3054
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@@ -212,11 -212,20 +213,22 @@@ function getDocumentStatusCode(doc: any
: null;
}
- export function estimateActualCredits(doc: any): number {
-function estimateActualCredits(doc: any, options: any): number {
- if (typeof doc?.metadata?.creditsUsed === "number") {
- return doc.metadata.creditsUsed;
++export function estimateActualCredits(doc: any, options?: any): number {
++ // Prefer the credits the scrape path actually recorded when present.
+ const creditsUsed = doc?.metadata?.creditsUsed;
- return typeof creditsUsed === "number" && Number.isFinite(creditsUsed)
- ? creditsUsed
- : 1;
++ if (typeof creditsUsed === "number" && Number.isFinite(creditsUsed)) {
++ return creditsUsed;
+ }
+ const formats = Array.isArray(options?.formats) ? options.formats : [];
+ // Only charge the JSON-extraction premium when extraction actually produced a
+ // document.json. When it failed, the page was still scraped, so fall back to
+ // the base scrape credit rather than billing for extraction that never ran.
+ // Deterministic JSON costs 7 (it generates a reusable extractor); plain JSON 5.
+ const producedJson = doc?.json != null;
+ if (!producedJson) return 1;
+ if (includesFormat(formats, "deterministicJson")) return 7;
+ if (includesFormat(formats, "json")) return 5;
+ return 1;
}
async function runSingleScrape(params: {
@@@ -276,7 -284,7 +288,7 @@@
return {
scrapeId,
doc,
- credits: estimateActualCredits(doc),
- credits: estimateActualCredits(doc, scrapeOptions),
++ credits: estimateActualCredits(doc, params.target.scrapeOptions),
};
}
@@@ -354,13 -356,9 +366,14 @@@ async function diffAndPersistPage(param
diff_text_bytes: diffTextBytes,
diff_json_bytes: diffJsonBytes,
status_code: getDocumentStatusCode(params.doc),
+ ...(error ? { error } : {}),
metadata: {
title: params.doc?.metadata?.title ?? null,
+ contentType: params.doc?.metadata?.contentType ?? null,
+ numPages: params.doc?.metadata?.numPages ?? null,
+ proxyUsed: params.doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,
+ creditsUsed: params.creditsUsed ?? null,
},
judgment,
emailStatus: status,
@@@ -543,9 -540,8 +556,9 @@@ async function runCrawlTarget(params:
const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];
if (!doc) continue;
const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);
- seen.add(hashMonitorUrl(url));
- const pageCredits = estimateActualCredits(doc);
+ seen.add(hashMonitorUrl(url).toString("hex"));
- credits += estimateActualCredits(doc, body.scrapeOptions);
++ const pageCredits = estimateActualCredits(doc, body.scrapeOptions);
+ credits += pageCredits;
pages.push(
await diffAndPersistPage({
monitor: params.monitor,
diff --cc apps/api/src/services/monitoring/store.ts
index 47b3b1e21,33cc09872..7994a0721
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@@ -1,7 -1,10 +1,10 @@@
import { createHash } from "crypto";
import { v7 as uuidv7 } from "uuid";
- import { supabase_rr_service, supabase_service } from "../supabase";
+ import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";
+ import { db, dbRr } from "../../db/connection";
+ import * as schema from "../../db/schema";
-import { includesFormat } from "../../lib/format-utils";
+ import { monitoringClaimDueMonitors } from "../../db/rpc";
+import { shouldParsePDF } from "../../controllers/v2/types";
import {
getNextMonitorRunAt,
estimateRunsPerMonth,
@@@ -29,122 -32,30 +32,128 @@@ function ensureTargetIds(targets: Array
})) as MonitorTarget[];
}
-// Per-page credit cost for a target's extraction mode. Mirrors
-// estimateActualCredits in runner.ts so the reserved/estimated credits match
-// what a check actually bills: deterministic JSON = 7, other JSON (plain or
-// changeTracking-json) = 5, otherwise the base scrape = 1.
-function perPageCredits(target: MonitorTarget): number {
- const formats = Array.isArray(target.scrapeOptions?.formats)
- ? (target.scrapeOptions!.formats as any[])
- : [];
- if (includesFormat(formats, "deterministicJson")) return 7;
- const hasJson =
- includesFormat(formats, "json") ||
- formats.some(
- format =>
- format?.type === "changeTracking" &&
- Array.isArray(format?.modes) &&
- format.modes.includes("json"),
- );
- return hasJson ? 5 : 1;
+const BASE_SCRAPE_CREDITS_PER_PAGE = 1;
+const JSON_SCRAPE_CREDITS_PER_PAGE = 5;
++const DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE = 7;
+const SCRAPE_OPTION_CREDIT_BONUS = 4;
+const JUDGE_CREDITS_PER_PAGE = 1;
+const REMOVED_PAGE_CREDITS = 0;
+const X_TWITTER_POSTPROCESSOR_CREDIT_BONUS = 29;
+const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;
+const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;
+
+type MonitorCreditMetadata = {
+ creditsUsed?: unknown;
+ numPages?: unknown;
+ proxyUsed?: unknown;
+ postprocessorsUsed?: unknown;
+};
+
+function formatType(format: unknown): string | null {
+ if (typeof format === "string") return format;
+ if (
+ format &&
+ typeof format === "object" &&
+ "type" in format &&
+ typeof format.type === "string"
+ ) {
+ return format.type;
+ }
+ return null;
+}
+
+function hasFormatOfType(formats: unknown, type: string): boolean {
+ return (
+ Array.isArray(formats) &&
+ formats.some(format => formatType(format) === type)
+ );
+}
+
+function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {
+ return types.some(type => hasFormatOfType(formats, type));
+}
+
+function requestsJsonChangeTracking(formats: unknown): boolean {
+ if (!Array.isArray(formats)) return false;
+ return formats.some(format => {
+ if (
+ !format ||
+ typeof format !== "object" ||
+ !("type" in format) ||
+ format.type !== "changeTracking"
+ ) {
+ return false;
+ }
+ const modes = "modes" in format ? format.modes : undefined;
+ return Array.isArray(modes) && modes.includes("json");
+ });
+}
+
+function estimateBaseCreditsPerPage(
+ options: MonitorTarget["scrapeOptions"],
+ params: { includeProxy?: boolean } = {},
+): number {
+ const formats = options?.formats;
+ const includeProxy = params.includeProxy ?? true;
++ const usesDeterministicJson = hasFormatOfType(formats, "deterministicJson");
+ const usesJsonCredits =
+ hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);
+ let credits = BASE_SCRAPE_CREDITS_PER_PAGE;
+
+ if (options?.lockdown) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
- if (usesJsonCredits) {
++ // Deterministic JSON generates a reusable extractor and costs more than plain
++ // JSON; both override the base scrape credit (mirrors estimateActualCredits).
++ if (usesDeterministicJson) {
++ credits = DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE;
++ } else if (usesJsonCredits) {
+ credits = JSON_SCRAPE_CREDITS_PER_PAGE;
+ }
+
+ if (hasAnyFormatOfType(formats, ["question", "query"])) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (hasFormatOfType(formats, "highlights")) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (hasFormatOfType(formats, "audio")) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (hasFormatOfType(formats, "video")) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (
+ includeProxy &&
+ (options?.proxy === "stealth" || options?.proxy === "enhanced")
+ ) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ return credits;
+}
+
+function estimateTargetBaseCredits(target: MonitorTarget): number {
+ const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);
+ if (target.type === "scrape") {
+ return target.urls.length * creditsPerPage;
+ }
+
+ const limit =
+ typeof target.crawlOptions?.limit === "number"
+ ? target.crawlOptions.limit
+ : DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE;
+ return Math.max(1, limit) * creditsPerPage;
}
-function estimateTargetCredits(target: MonitorTarget): number {
- const perPage = perPageCredits(target);
+function estimateTargetPageCount(target: MonitorTarget): number {
if (target.type === "scrape") {
- return target.urls.length * perPage;
+ return target.urls.length;
}
const limit =
@@@ -753,62 -624,26 +824,60 @@@ export async function countMonitorCheck
targetId?: string;
status?: string;
}): Promise<number> {
- const pageSize = 1000;
- let total = 0;
- let offset = 0;
-
- while (true) {
- let query = supabase_rr_service
- .from("monitor_check_pages")
- .select("id")
- .eq("check_id", params.checkId);
-
- if (params.targetId) {
- query = query.eq("target_id", params.targetId);
- }
- if (params.status) {
- query = query.eq("status", params.status);
- }
-
- const { data, error } = await query.range(offset, offset + pageSize - 1);
- throwIfError(error, "Failed to count monitor check pages");
-
- const batch = data ?? [];
- total += batch.length;
- if (batch.length < pageSize) break;
- offset += pageSize;
+ const conditions = [eq(schema.monitor_check_pages.check_id, params.checkId)];
+ if (params.targetId) {
+ conditions.push(eq(schema.monitor_check_pages.target_id, params.targetId));
+ }
+ if (params.status) {
+ conditions.push(eq(schema.monitor_check_pages.status, params.status));
}
- return total;
+ const [row] = await run(
+ () =>
+ dbRr
+ .select({ value: count() })
+ .from(schema.monitor_check_pages)
+ .where(and(...conditions)),
+ "Failed to count monitor check pages",
+ );
+
+ return row?.value ?? 0;
}
+export async function calculateMonitorCheckActualCredits(params: {
+ checkId: string;
+ targets: MonitorTarget[];
+}): Promise<number> {
+ let total = 0;
+ let offset = 0;
+
+ while (true) {
- const { data, error } = await supabase_rr_service
- .from("monitor_check_pages")
- .select("target_id, metadata, judgment, status")
- .eq("check_id", params.checkId)
- .order("id", { ascending: true })
- .range(offset, offset + MONITOR_CHECK_PAGE_BATCH_SIZE - 1);
++ const batch = await run(
++ () =>
++ dbRr
++ .select({
++ target_id: schema.monitor_check_pages.target_id,
++ metadata: schema.monitor_check_pages.metadata,
++ judgment: schema.monitor_check_pages.judgment,
++ status: schema.monitor_check_pages.status,
++ })
++ .from(schema.monitor_check_pages)
++ .where(eq(schema.monitor_check_pages.check_id, params.checkId))
++ .orderBy(asc(schema.monitor_check_pages.id))
++ .limit(MONITOR_CHECK_PAGE_BATCH_SIZE)
++ .offset(offset),
++ "Failed to calculate monitor check credits",
++ );
+
- throwIfError(error, "Failed to calculate monitor check credits");
-
- const batch = data ?? [];
+ total += calculateMonitorCheckActualCreditsFromPages(batch, params.targets);
+
+ if (batch.length < MONITOR_CHECK_PAGE_BATCH_SIZE) break;
+ offset += MONITOR_CHECK_PAGE_BATCH_SIZE;
+ }
+
+ return total;
+}
+
export async function getMonitorPage(params: {
monitorId: string;
targetId: string;