Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add consistentRead option for monitor DB reads

Developers Digest4ea3a3c71225
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index b5aee33a4..2e767103e 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -121,6 +121,36 @@ describe("monitoring store credit helpers", () => {
).toBe(34);
});
+ it("does not add fallback proxy credits when runtime metadata says basic was used", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://example.com/sample.pdf"],
+ scrapeOptions: {
+ formats: [{ type: "changeTracking", modes: ["json"] }],
+ proxy: "stealth",
+ },
+ },
+ ];
+
+ expect(
+ calculateMonitorCheckActualCreditsFromPages(
+ [
+ {
+ target_id: "target-1",
+ metadata: {
+ numPages: 10,
+ proxyUsed: "basic",
+ },
+ status: "same",
+ },
+ ],
+ targets,
+ ),
+ ).toBe(14);
+ });
+
it("prefers recorded page usage and does not bill removed pages", () => {
expect(
calculateMonitorCheckActualCreditsFromPages([
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index dd6603d6f..01c026190 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -87,8 +87,10 @@ function requestsJsonChangeTracking(formats: unknown): boolean {
function estimateBaseCreditsPerPage(
options: MonitorTarget["scrapeOptions"],
+ params: { includeProxy?: boolean } = {},
): number {
const formats = options?.formats;
+ const includeProxy = params.includeProxy ?? true;
const usesJsonCredits =
hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);
let credits = BASE_SCRAPE_CREDITS_PER_PAGE;
@@ -117,7 +119,10 @@ function estimateBaseCreditsPerPage(
credits += SCRAPE_OPTION_CREDIT_BONUS;
}
- if (options?.proxy === "stealth" || options?.proxy === "enhanced") {
+ if (
+ includeProxy &&
+ (options?.proxy === "stealth" || options?.proxy === "enhanced")
+ ) {
credits += SCRAPE_OPTION_CREDIT_BONUS;
}
@@ -179,7 +184,9 @@ export function calculateMonitorCheckActualCreditsFromPages(
const baseCreditsByTarget = new Map(
targets.map(target => [
target.id,
- estimateBaseCreditsPerPage(target.scrapeOptions),
+ estimateBaseCreditsPerPage(target.scrapeOptions, {
+ includeProxy: false,
+ }),
]),
);
const targetsById = new Map(targets.map(target => [target.id, target]));
@@ -211,10 +218,12 @@ export function calculateMonitorCheckActualCreditsFromPages(
credits += metadata.numPages - 1;
}
+ const requestedPremiumProxy =
+ target?.scrapeOptions?.proxy === "stealth" ||
+ target?.scrapeOptions?.proxy === "enhanced";
if (
- metadata?.proxyUsed === "stealth" &&
- target?.scrapeOptions?.proxy !== "stealth" &&
- target?.scrapeOptions?.proxy !== "enhanced"
+ metadata?.proxyUsed === "stealth" ||
+ (metadata?.proxyUsed == null && requestedPremiumProxy)
) {
credits += SCRAPE_OPTION_CREDIT_BONUS;
}