Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
merge main: re-port monitor billing onto drizzle + deterministicjson
Developers Digest281c9717e7fd
diff --cc apps/api/src/services/monitoring/results.tsindex 2641ef741,7ba63ef99..1dd663822--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@@ -194,12 -194,9 +195,13 @@@ export async function recordMonitorScradiff_text_bytes: diffTextBytes,diff_json_bytes: diffJsonBytes,status_code: getDocumentStatusCode(doc),+ ...(error ? { error } : {}),metadata: {title: doc?.metadata?.title ?? null,+ contentType: doc?.metadata?.contentType ?? null,+ numPages: doc?.metadata?.numPages ?? null,+ proxyUsed: doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,creditsUsed: doc?.metadata?.creditsUsed ?? null,},judgment: judgment ?? null,diff --cc apps/api/src/services/monitoring/runner.tsindex ae26cf328,58354e9fa..5b73f3054--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@@ -212,11 -212,20 +213,22 @@@ function getDocumentStatusCode(doc: any: null;}- export function estimateActualCredits(doc: any): number {-function estimateActualCredits(doc: any, options: any): number {- if (typeof doc?.metadata?.creditsUsed === "number") {- return doc.metadata.creditsUsed;++export function estimateActualCredits(doc: any, options?: any): number {++ // Prefer the credits the scrape path actually recorded when present.+ const creditsUsed = doc?.metadata?.creditsUsed;- return typeof creditsUsed === "number" && Number.isFinite(creditsUsed)- ? creditsUsed- : 1;++ if (typeof creditsUsed === "number" && Number.isFinite(creditsUsed)) {++ return creditsUsed;+ }+ const formats = Array.isArray(options?.formats) ? options.formats : [];+ // Only charge the JSON-extraction premium when extraction actually produced a+ // document.json. When it failed, the page was still scraped, so fall back to+ // the base scrape credit rather than billing for extraction that never ran.+ // Deterministic JSON costs 7 (it generates a reusable extractor); plain JSON 5.+ const producedJson = doc?.json != null;+ if (!producedJson) return 1;+ if (includesFormat(formats, "deterministicJson")) return 7;+ if (includesFormat(formats, "json")) return 5;+ return 1;}async function runSingleScrape(params: {@@@ -276,7 -284,7 +288,7 @@@return {scrapeId,doc,- credits: estimateActualCredits(doc),- credits: estimateActualCredits(doc, scrapeOptions),++ credits: estimateActualCredits(doc, params.target.scrapeOptions),};}@@@ -354,13 -356,9 +366,14 @@@ async function diffAndPersistPage(paramdiff_text_bytes: diffTextBytes,diff_json_bytes: diffJsonBytes,status_code: getDocumentStatusCode(params.doc),+ ...(error ? { error } : {}),metadata: {title: params.doc?.metadata?.title ?? null,+ contentType: params.doc?.metadata?.contentType ?? null,+ numPages: params.doc?.metadata?.numPages ?? null,+ proxyUsed: params.doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,+ creditsUsed: params.creditsUsed ?? null,},judgment,emailStatus: status,@@@ -543,9 -540,8 +556,9 @@@ async function runCrawlTarget(params:const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];if (!doc) continue;const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);- seen.add(hashMonitorUrl(url));- const pageCredits = estimateActualCredits(doc);+ seen.add(hashMonitorUrl(url).toString("hex"));- credits += estimateActualCredits(doc, body.scrapeOptions);++ const pageCredits = estimateActualCredits(doc, body.scrapeOptions);+ credits += pageCredits;pages.push(await diffAndPersistPage({monitor: params.monitor,diff --cc apps/api/src/services/monitoring/store.tsindex 47b3b1e21,33cc09872..7994a0721--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@@ -1,7 -1,10 +1,10 @@@import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";- import { supabase_rr_service, supabase_service } from "../supabase";+ import { and, asc, count, desc, eq, isNull, ne } from "drizzle-orm";+ import { db, dbRr } from "../../db/connection";+ import * as schema from "../../db/schema";-import { includesFormat } from "../../lib/format-utils";+ import { monitoringClaimDueMonitors } from "../../db/rpc";+import { shouldParsePDF } from "../../controllers/v2/types";import {getNextMonitorRunAt,estimateRunsPerMonth,@@@ -29,122 -32,30 +32,128 @@@ function ensureTargetIds(targets: Array})) as MonitorTarget[];}-// Per-page credit cost for a target's extraction mode. Mirrors-// estimateActualCredits in runner.ts so the reserved/estimated credits match-// what a check actually bills: deterministic JSON = 7, other JSON (plain or-// changeTracking-json) = 5, otherwise the base scrape = 1.-function perPageCredits(target: MonitorTarget): number {- const formats = Array.isArray(target.scrapeOptions?.formats)- ? (target.scrapeOptions!.formats as any[])- : [];- if (includesFormat(formats, "deterministicJson")) return 7;- const hasJson =- includesFormat(formats, "json") ||- formats.some(- format =>- format?.type === "changeTracking" &&- Array.isArray(format?.modes) &&- format.modes.includes("json"),- );- return hasJson ? 5 : 1;+const BASE_SCRAPE_CREDITS_PER_PAGE = 1;+const JSON_SCRAPE_CREDITS_PER_PAGE = 5;++const DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE = 7;+const SCRAPE_OPTION_CREDIT_BONUS = 4;+const JUDGE_CREDITS_PER_PAGE = 1;+const REMOVED_PAGE_CREDITS = 0;+const X_TWITTER_POSTPROCESSOR_CREDIT_BONUS = 29;+const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;+const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;++type MonitorCreditMetadata = {+ creditsUsed?: unknown;+ numPages?: unknown;+ proxyUsed?: unknown;+ postprocessorsUsed?: unknown;+};++function formatType(format: unknown): string | null {+ if (typeof format === "string") return format;+ if (+ format &&+ typeof format === "object" &&+ "type" in format &&+ typeof format.type === "string"+ ) {+ return format.type;+ }+ return null;+}++function hasFormatOfType(formats: unknown, type: string): boolean {+ return (+ Array.isArray(formats) &&+ formats.some(format => formatType(format) === type)+ );+}++function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {+ return types.some(type => hasFormatOfType(formats, type));+}++function requestsJsonChangeTracking(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ return formats.some(format => {+ if (+ !format ||+ typeof format !== "object" ||+ !("type" in format) ||+ format.type !== "changeTracking"+ ) {+ return false;+ }+ const modes = "modes" in format ? format.modes : undefined;+ return Array.isArray(modes) && modes.includes("json");+ });+}++function estimateBaseCreditsPerPage(+ options: MonitorTarget["scrapeOptions"],+ params: { includeProxy?: boolean } = {},+): number {+ const formats = options?.formats;+ const includeProxy = params.includeProxy ?? true;++ const usesDeterministicJson = hasFormatOfType(formats, "deterministicJson");+ const usesJsonCredits =+ hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);+ let credits = BASE_SCRAPE_CREDITS_PER_PAGE;++ if (options?.lockdown) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }+- if (usesJsonCredits) {++ // Deterministic JSON generates a reusable extractor and costs more than plain++ // JSON; both override the base scrape credit (mirrors estimateActualCredits).++ if (usesDeterministicJson) {++ credits = DETERMINISTIC_JSON_SCRAPE_CREDITS_PER_PAGE;++ } else if (usesJsonCredits) {+ credits = JSON_SCRAPE_CREDITS_PER_PAGE;+ }++ if (hasAnyFormatOfType(formats, ["question", "query"])) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (hasFormatOfType(formats, "highlights")) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (hasFormatOfType(formats, "audio")) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (hasFormatOfType(formats, "video")) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (+ includeProxy &&+ (options?.proxy === "stealth" || options?.proxy === "enhanced")+ ) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ return credits;+}++function estimateTargetBaseCredits(target: MonitorTarget): number {+ const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);+ if (target.type === "scrape") {+ return target.urls.length * creditsPerPage;+ }++ const limit =+ typeof target.crawlOptions?.limit === "number"+ ? target.crawlOptions.limit+ : DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE;+ return Math.max(1, limit) * creditsPerPage;}-function estimateTargetCredits(target: MonitorTarget): number {- const perPage = perPageCredits(target);+function estimateTargetPageCount(target: MonitorTarget): number {if (target.type === "scrape") {- return target.urls.length * perPage;+ return target.urls.length;}const limit =@@@ -753,62 -624,26 +824,60 @@@ export async function countMonitorChecktargetId?: string;status?: string;}): Promise<number> {- const pageSize = 1000;- let total = 0;- let offset = 0;-- while (true) {- let query = supabase_rr_service- .from("monitor_check_pages")- .select("id")- .eq("check_id", params.checkId);-- if (params.targetId) {- query = query.eq("target_id", params.targetId);- }- if (params.status) {- query = query.eq("status", params.status);- }-- const { data, error } = await query.range(offset, offset + pageSize - 1);- throwIfError(error, "Failed to count monitor check pages");-- const batch = data ?? [];- total += batch.length;- if (batch.length < pageSize) break;- offset += pageSize;+ const conditions = [eq(schema.monitor_check_pages.check_id, params.checkId)];+ if (params.targetId) {+ conditions.push(eq(schema.monitor_check_pages.target_id, params.targetId));+ }+ if (params.status) {+ conditions.push(eq(schema.monitor_check_pages.status, params.status));}- return total;+ const [row] = await run(+ () =>+ dbRr+ .select({ value: count() })+ .from(schema.monitor_check_pages)+ .where(and(...conditions)),+ "Failed to count monitor check pages",+ );++ return row?.value ?? 0;}+export async function calculateMonitorCheckActualCredits(params: {+ checkId: string;+ targets: MonitorTarget[];+}): Promise<number> {+ let total = 0;+ let offset = 0;++ while (true) {- const { data, error } = await supabase_rr_service- .from("monitor_check_pages")- .select("target_id, metadata, judgment, status")- .eq("check_id", params.checkId)- .order("id", { ascending: true })- .range(offset, offset + MONITOR_CHECK_PAGE_BATCH_SIZE - 1);++ const batch = await run(++ () =>++ dbRr++ .select({++ target_id: schema.monitor_check_pages.target_id,++ metadata: schema.monitor_check_pages.metadata,++ judgment: schema.monitor_check_pages.judgment,++ status: schema.monitor_check_pages.status,++ })++ .from(schema.monitor_check_pages)++ .where(eq(schema.monitor_check_pages.check_id, params.checkId))++ .orderBy(asc(schema.monitor_check_pages.id))++ .limit(MONITOR_CHECK_PAGE_BATCH_SIZE)++ .offset(offset),++ "Failed to calculate monitor check credits",++ );+- throwIfError(error, "Failed to calculate monitor check credits");-- const batch = data ?? [];+ total += calculateMonitorCheckActualCreditsFromPages(batch, params.targets);++ if (batch.length < MONITOR_CHECK_PAGE_BATCH_SIZE) break;+ offset += MONITOR_CHECK_PAGE_BATCH_SIZE;+ }++ return total;+}+export async function getMonitorPage(params: {monitorId: string;targetId: string;