Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add consistentRead option for monitor DB reads
Developers Digest4ea3a3c71225
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex b5aee33a4..2e767103e 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -121,6 +121,36 @@ describe("monitoring store credit helpers", () => {).toBe(34);});+ it("does not add fallback proxy credits when runtime metadata says basic was used", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/sample.pdf"],+ scrapeOptions: {+ formats: [{ type: "changeTracking", modes: ["json"] }],+ proxy: "stealth",+ },+ },+ ];++ expect(+ calculateMonitorCheckActualCreditsFromPages(+ [+ {+ target_id: "target-1",+ metadata: {+ numPages: 10,+ proxyUsed: "basic",+ },+ status: "same",+ },+ ],+ targets,+ ),+ ).toBe(14);+ });+it("prefers recorded page usage and does not bill removed pages", () => {expect(calculateMonitorCheckActualCreditsFromPages([diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex dd6603d6f..01c026190 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -87,8 +87,10 @@ function requestsJsonChangeTracking(formats: unknown): boolean {function estimateBaseCreditsPerPage(options: MonitorTarget["scrapeOptions"],+ params: { includeProxy?: boolean } = {},): number {const formats = options?.formats;+ const includeProxy = params.includeProxy ?? true;const usesJsonCredits =hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);let credits = BASE_SCRAPE_CREDITS_PER_PAGE;@@ -117,7 +119,10 @@ function estimateBaseCreditsPerPage(credits += SCRAPE_OPTION_CREDIT_BONUS;}- if (options?.proxy === "stealth" || options?.proxy === "enhanced") {+ if (+ includeProxy &&+ (options?.proxy === "stealth" || options?.proxy === "enhanced")+ ) {credits += SCRAPE_OPTION_CREDIT_BONUS;}@@ -179,7 +184,9 @@ export function calculateMonitorCheckActualCreditsFromPages(const baseCreditsByTarget = new Map(targets.map(target => [target.id,- estimateBaseCreditsPerPage(target.scrapeOptions),+ estimateBaseCreditsPerPage(target.scrapeOptions, {+ includeProxy: false,+ }),]),);const targetsById = new Map(targets.map(target => [target.id, target]));@@ -211,10 +218,12 @@ export function calculateMonitorCheckActualCreditsFromPages(credits += metadata.numPages - 1;}+ const requestedPremiumProxy =+ target?.scrapeOptions?.proxy === "stealth" ||+ target?.scrapeOptions?.proxy === "enhanced";if (- metadata?.proxyUsed === "stealth" &&- target?.scrapeOptions?.proxy !== "stealth" &&- target?.scrapeOptions?.proxy !== "enhanced"+ metadata?.proxyUsed === "stealth" ||+ (metadata?.proxyUsed == null && requestedPremiumProxy)) {credits += SCRAPE_OPTION_CREDIT_BONUS;}