Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Include the x-twitter surcharge in monitor credit estimates (#4841)
Rakshith Ramprakashaaf95e6993b1
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsindex 2d37404f4..10eafd98f 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -6,6 +6,14 @@ import { Meta } from "../..";import { EngineScrapeResult } from "..";import { EngineError, XTwitterConfigurationError } from "../../error";import { safeMarkdownToHtml } from "../pdf/markdownToHtml";+import {+ isXTwitterUrl,+ parseXTwitterUrl,+ type XTwitterPostUrl,+ type XTwitterProfileUrl,+} from "./url";++export { isXTwitterUrl };const XAI_RESPONSES_MODEL = "grok-4-1-fast-non-reasoning";@@ -21,37 +29,6 @@ function xTwitterTelemetry(functionId: string, meta: Meta) {};}-const RESERVED_PROFILE_PATHS = new Set([- "compose",- "explore",- "hashtag",- "home",- "i",- "intent",- "login",- "logout",- "messages",- "notifications",- "search",- "settings",- "share",-]);--type XTwitterProfileUrl = {- kind: "profile";- handle: string;- normalizedUrl: string;-};--type XTwitterPostUrl = {- kind: "post";- handle?: string;- postId: string;- normalizedUrl: string;-};--type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;-type ProfilePost = {text: string;url?: string | null;@@ -220,99 +197,6 @@ const postSchema = {additionalProperties: false,};-function parseXTwitterUrl(url: string): XTwitterUrl | null {- let parsed: URL;- try {- parsed = new URL(url);- } catch {- return null;- }-- if (!["http:", "https:"].includes(parsed.protocol)) {- return null;- }-- const hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");- if (- hostname !== "x.com" &&- hostname !== "twitter.com" &&- hostname !== "mobile.twitter.com"- ) {- return null;- }-- const segments = parsed.pathname- .split("/")- .map(segment => segment.trim())- .filter(Boolean);-- if (segments.length === 0) {- return null;- }-- if (- segments.length >= 4 &&- segments[0] === "i" &&- segments[1] === "web" &&- segments[2] === "status" &&- isPostId(segments[3])- ) {- return {- kind: "post",- postId: segments[3],- normalizedUrl: `https://x.com/i/web/status/${segments[3]}`,- };- }-- if (- segments.length >= 3 &&- segments[0] === "i" &&- segments[1] === "status" &&- isPostId(segments[2])- ) {- return {- kind: "post",- postId: segments[2],- normalizedUrl: `https://x.com/i/web/status/${segments[2]}`,- };- }-- if (- segments.length >= 3 &&- isHandle(segments[0]) &&- ["status", "statuses"].includes(segments[1]) &&- isPostId(segments[2])- ) {- const handle = segments[0];- const postId = segments[2];- return {- kind: "post",- handle,- postId,- normalizedUrl: `https://x.com/${handle}/status/${postId}`,- };- }-- if (- segments.length === 1 &&- isHandle(segments[0]) &&- !RESERVED_PROFILE_PATHS.has(segments[0].toLowerCase())- ) {- const handle = segments[0];- return {- kind: "profile",- handle,- normalizedUrl: `https://x.com/${handle}`,- };- }-- return null;-}--export function isXTwitterUrl(url: string): boolean {- return parseXTwitterUrl(url) !== null;-}-export async function scrapeURLWithXTwitter(meta: Meta,): Promise<EngineScrapeResult> {@@ -582,14 +466,6 @@ async function fetchPost(return output as XTwitterPostData;}-function isHandle(value: string): boolean {- return /^[A-Za-z0-9_]{1,15}$/.test(value);-}--function isPostId(value: string): boolean {- return /^\d{5,}$/.test(value);-}-function stripAt(value: string | null | undefined): string | undefined {const trimmed = value?.trim();if (!trimmed) {diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.tsnew file mode 100644index 000000000..09539e810--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts@@ -0,0 +1,134 @@+// Pure URL recognition for the X/Twitter engine, kept apart from the engine so+// credit estimation can use it without pulling in the scraper and AI SDK.++const RESERVED_PROFILE_PATHS = new Set([+ "compose",+ "explore",+ "hashtag",+ "home",+ "i",+ "intent",+ "login",+ "logout",+ "messages",+ "notifications",+ "search",+ "settings",+ "share",+]);++export type XTwitterProfileUrl = {+ kind: "profile";+ handle: string;+ normalizedUrl: string;+};++export type XTwitterPostUrl = {+ kind: "post";+ handle?: string;+ postId: string;+ normalizedUrl: string;+};++export type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;++export function parseXTwitterUrl(url: string): XTwitterUrl | null {+ let parsed: URL;+ try {+ parsed = new URL(url);+ } catch {+ return null;+ }++ if (!["http:", "https:"].includes(parsed.protocol)) {+ return null;+ }++ const hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");+ if (+ hostname !== "x.com" &&+ hostname !== "twitter.com" &&+ hostname !== "mobile.twitter.com"+ ) {+ return null;+ }++ const segments = parsed.pathname+ .split("/")+ .map(segment => segment.trim())+ .filter(Boolean);++ if (segments.length === 0) {+ return null;+ }++ if (+ segments.length >= 4 &&+ segments[0] === "i" &&+ segments[1] === "web" &&+ segments[2] === "status" &&+ isPostId(segments[3])+ ) {+ return {+ kind: "post",+ postId: segments[3],+ normalizedUrl: `https://x.com/i/web/status/${segments[3]}`,+ };+ }++ if (+ segments.length >= 3 &&+ segments[0] === "i" &&+ segments[1] === "status" &&+ isPostId(segments[2])+ ) {+ return {+ kind: "post",+ postId: segments[2],+ normalizedUrl: `https://x.com/i/web/status/${segments[2]}`,+ };+ }++ if (+ segments.length >= 3 &&+ isHandle(segments[0]) &&+ ["status", "statuses"].includes(segments[1]) &&+ isPostId(segments[2])+ ) {+ const handle = segments[0];+ const postId = segments[2];+ return {+ kind: "post",+ handle,+ postId,+ normalizedUrl: `https://x.com/${handle}/status/${postId}`,+ };+ }++ if (+ segments.length === 1 &&+ isHandle(segments[0]) &&+ !RESERVED_PROFILE_PATHS.has(segments[0].toLowerCase())+ ) {+ const handle = segments[0];+ return {+ kind: "profile",+ handle,+ normalizedUrl: `https://x.com/${handle}`,+ };+ }++ return null;+}++export function isXTwitterUrl(url: string): boolean {+ return parseXTwitterUrl(url) !== null;+}++function isHandle(value: string): boolean {+ return /^[A-Za-z0-9_]{1,15}$/.test(value);+}++function isPostId(value: string): boolean {+ return /^\d{5,}$/.test(value);+}diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex c06003e2e..900220194 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -9,6 +9,7 @@ import {} from "./store";import { judgeCreditsForJudgedCount } from "./search/billing";import type { MonitorTarget } from "./types";+import { config } from "../../config";describe("monitoring store credit helpers", () => {it("estimates goal-enabled scrape monitors from scrape option costs", () => {@@ -29,6 +30,108 @@ describe("monitoring store credit helpers", () => {expect(estimateMonitorCreditsPerRun(targets, true)).toBe(12);});+ // The x-twitter engine only exists with an xAI key or DB auth.+ const withXTwitterEngine = (fn: () => void) => {+ const saved = config.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = true;+ try {+ fn();+ } finally {+ config.USE_DB_AUTHENTICATION = saved;+ }+ };++ it("includes the x-twitter surcharge, so the hold covers what is billed", () =>+ withXTwitterEngine(() => {+ const targets: MonitorTarget[] = [+ {+ id: "target-x",+ type: "scrape",+ urls: ["https://x.com/blknoiz06"],+ scrapeOptions: {},+ },+ ];++ // 1 scrape + 29 x-twitter, + 1 judge.+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(30);+ expect(estimateMonitorCreditsPerRun(targets, true)).toBe(31);+ expect(+ calculateMonitorCheckActualCreditsFromPages(+ [+ {+ target_id: "target-x",+ status: "changed",+ metadata: {+ creditsUsed: null,+ postprocessorsUsed: ["x-twitter"],+ },+ judgment: { meaningful: false },+ },+ ],+ targets,+ ),+ ).toBe(estimateMonitorCreditsPerRun(targets, true));+ }));++ it("charges the surcharge only for URLs the x-twitter engine takes", () =>+ withXTwitterEngine(() => {+ const targets: MonitorTarget[] = [+ {+ id: "target-mixed",+ type: "scrape",+ urls: [+ "https://twitter.com/someone/status/1234567890",+ "https://x.com/home",+ "https://example.com/x.com",+ ],+ scrapeOptions: {},+ },+ ];++ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(3 + 29);+ }));++ it("skips the surcharge when a browser profile bypasses the x-twitter engine", () =>+ withXTwitterEngine(() => {+ const targets: MonitorTarget[] = [+ {+ id: "target-profile",+ type: "scrape",+ urls: ["https://x.com/someone"],+ scrapeOptions: { profile: { name: "p" } },+ },+ ];++ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(1);+ }));++ it("skips the surcharge where the x-twitter engine cannot run", () => {+ const target = (+ scrapeOptions: Record<string, unknown>,+ ): MonitorTarget[] => [+ {+ id: "t",+ type: "scrape",+ urls: ["https://x.com/someone"],+ scrapeOptions,+ },+ ];+ withXTwitterEngine(() => {+ // Lockdown serves from the index: 1 base + 4 lockdown, no surcharge.+ expect(+ estimateMonitorCreditsPerRun(target({ lockdown: true }), false),+ ).toBe(5);+ });+ const saved = [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] as const;+ config.USE_DB_AUTHENTICATION = false;+ config.XAI_API_KEY = undefined;+ try {+ expect(estimateMonitorCreditsPerRun(target({}), false)).toBe(1);+ } finally {+ [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] = saved;+ }+ });+it("adds predictable lockdown costs and judge credits separately", () => {const targets: MonitorTarget[] = [{diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 9c9f31211..94ec5a054 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -4,7 +4,9 @@ import { and, asc, count, desc, eq, isNull, ne, sql } from "drizzle-orm";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { monitoringClaimDueMonitors } from "../../db/rpc";+import { config } from "../../config";import { shouldParsePDF } from "../../controllers/v2/types";+import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";import {getNextMonitorRunAt,estimateRunsPerMonth,@@ -173,13 +175,37 @@ function estimateSearchTargetCredits();}+/**+ * The x-twitter engine's surcharge, which billing adds per page (see+ * fallbackBaseCreditsForPage). Left out of the estimate, a judged one-URL X+ * monitor reserves 2 credits and costs 31, and an account with a few credits+ * left passes every hold and is never charged.+ */+function xTwitterSurcharge(+ url: string,+ options: MonitorTarget["scrapeOptions"],+): number {+ // Mirrors the engine router (scrapeURL/engines/index.ts): the engine exists+ // only with an xAI key or DB auth, lockdown serves from the index alone, and+ // a browser profile routes around it.+ const engineEnabled =+ (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||+ config.USE_DB_AUTHENTICATION === true;+ if (!engineEnabled || options?.lockdown || options?.profile) return 0;+ return isXTwitterUrl(url) ? X_TWITTER_POSTPROCESSOR_CREDIT_BONUS : 0;+}+function estimateTargetBaseCredits(target: MonitorTarget,judgeEnabled: boolean = false,): number {const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);if (target.type === "scrape") {- return target.urls.length * creditsPerPage;+ return target.urls.reduce(+ (sum, url) =>+ sum + creditsPerPage + xTwitterSurcharge(url, target.scrapeOptions),+ 0,+ );}if (target.type === "search") {return estimateSearchTargetCredits(target, judgeEnabled);