Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Include the x-twitter surcharge in monitor credit estimates (#4841)

Rakshith Ramprakashaaf95e6993b1
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
index 2d37404f4..10eafd98f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
@@ -6,6 +6,14 @@ import { Meta } from "../..";
import { EngineScrapeResult } from "..";
import { EngineError, XTwitterConfigurationError } from "../../error";
import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
+import {
+ isXTwitterUrl,
+ parseXTwitterUrl,
+ type XTwitterPostUrl,
+ type XTwitterProfileUrl,
+} from "./url";
+
+export { isXTwitterUrl };
const XAI_RESPONSES_MODEL = "grok-4-1-fast-non-reasoning";
@@ -21,37 +29,6 @@ function xTwitterTelemetry(functionId: string, meta: Meta) {
};
}
-const RESERVED_PROFILE_PATHS = new Set([
- "compose",
- "explore",
- "hashtag",
- "home",
- "i",
- "intent",
- "login",
- "logout",
- "messages",
- "notifications",
- "search",
- "settings",
- "share",
-]);
-
-type XTwitterProfileUrl = {
- kind: "profile";
- handle: string;
- normalizedUrl: string;
-};
-
-type XTwitterPostUrl = {
- kind: "post";
- handle?: string;
- postId: string;
- normalizedUrl: string;
-};
-
-type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;
-
type ProfilePost = {
text: string;
url?: string | null;
@@ -220,99 +197,6 @@ const postSchema = {
additionalProperties: false,
};
-function parseXTwitterUrl(url: string): XTwitterUrl | null {
- let parsed: URL;
- try {
- parsed = new URL(url);
- } catch {
- return null;
- }
-
- if (!["http:", "https:"].includes(parsed.protocol)) {
- return null;
- }
-
- const hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");
- if (
- hostname !== "x.com" &&
- hostname !== "twitter.com" &&
- hostname !== "mobile.twitter.com"
- ) {
- return null;
- }
-
- const segments = parsed.pathname
- .split("/")
- .map(segment => segment.trim())
- .filter(Boolean);
-
- if (segments.length === 0) {
- return null;
- }
-
- if (
- segments.length >= 4 &&
- segments[0] === "i" &&
- segments[1] === "web" &&
- segments[2] === "status" &&
- isPostId(segments[3])
- ) {
- return {
- kind: "post",
- postId: segments[3],
- normalizedUrl: `https://x.com/i/web/status/${segments[3]}`,
- };
- }
-
- if (
- segments.length >= 3 &&
- segments[0] === "i" &&
- segments[1] === "status" &&
- isPostId(segments[2])
- ) {
- return {
- kind: "post",
- postId: segments[2],
- normalizedUrl: `https://x.com/i/web/status/${segments[2]}`,
- };
- }
-
- if (
- segments.length >= 3 &&
- isHandle(segments[0]) &&
- ["status", "statuses"].includes(segments[1]) &&
- isPostId(segments[2])
- ) {
- const handle = segments[0];
- const postId = segments[2];
- return {
- kind: "post",
- handle,
- postId,
- normalizedUrl: `https://x.com/${handle}/status/${postId}`,
- };
- }
-
- if (
- segments.length === 1 &&
- isHandle(segments[0]) &&
- !RESERVED_PROFILE_PATHS.has(segments[0].toLowerCase())
- ) {
- const handle = segments[0];
- return {
- kind: "profile",
- handle,
- normalizedUrl: `https://x.com/${handle}`,
- };
- }
-
- return null;
-}
-
-export function isXTwitterUrl(url: string): boolean {
- return parseXTwitterUrl(url) !== null;
-}
-
export async function scrapeURLWithXTwitter(
meta: Meta,
): Promise<EngineScrapeResult> {
@@ -582,14 +466,6 @@ async function fetchPost(
return output as XTwitterPostData;
}
-function isHandle(value: string): boolean {
- return /^[A-Za-z0-9_]{1,15}$/.test(value);
-}
-
-function isPostId(value: string): boolean {
- return /^\d{5,}$/.test(value);
-}
-
function stripAt(value: string | null | undefined): string | undefined {
const trimmed = value?.trim();
if (!trimmed) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
new file mode 100644
index 000000000..09539e810
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/url.ts
@@ -0,0 +1,134 @@
+// Pure URL recognition for the X/Twitter engine, kept apart from the engine so
+// credit estimation can use it without pulling in the scraper and AI SDK.
+
+const RESERVED_PROFILE_PATHS = new Set([
+ "compose",
+ "explore",
+ "hashtag",
+ "home",
+ "i",
+ "intent",
+ "login",
+ "logout",
+ "messages",
+ "notifications",
+ "search",
+ "settings",
+ "share",
+]);
+
+export type XTwitterProfileUrl = {
+ kind: "profile";
+ handle: string;
+ normalizedUrl: string;
+};
+
+export type XTwitterPostUrl = {
+ kind: "post";
+ handle?: string;
+ postId: string;
+ normalizedUrl: string;
+};
+
+export type XTwitterUrl = XTwitterProfileUrl | XTwitterPostUrl;
+
+export function parseXTwitterUrl(url: string): XTwitterUrl | null {
+ let parsed: URL;
+ try {
+ parsed = new URL(url);
+ } catch {
+ return null;
+ }
+
+ if (!["http:", "https:"].includes(parsed.protocol)) {
+ return null;
+ }
+
+ const hostname = parsed.hostname.toLowerCase().replace(/^www\./, "");
+ if (
+ hostname !== "x.com" &&
+ hostname !== "twitter.com" &&
+ hostname !== "mobile.twitter.com"
+ ) {
+ return null;
+ }
+
+ const segments = parsed.pathname
+ .split("/")
+ .map(segment => segment.trim())
+ .filter(Boolean);
+
+ if (segments.length === 0) {
+ return null;
+ }
+
+ if (
+ segments.length >= 4 &&
+ segments[0] === "i" &&
+ segments[1] === "web" &&
+ segments[2] === "status" &&
+ isPostId(segments[3])
+ ) {
+ return {
+ kind: "post",
+ postId: segments[3],
+ normalizedUrl: `https://x.com/i/web/status/${segments[3]}`,
+ };
+ }
+
+ if (
+ segments.length >= 3 &&
+ segments[0] === "i" &&
+ segments[1] === "status" &&
+ isPostId(segments[2])
+ ) {
+ return {
+ kind: "post",
+ postId: segments[2],
+ normalizedUrl: `https://x.com/i/web/status/${segments[2]}`,
+ };
+ }
+
+ if (
+ segments.length >= 3 &&
+ isHandle(segments[0]) &&
+ ["status", "statuses"].includes(segments[1]) &&
+ isPostId(segments[2])
+ ) {
+ const handle = segments[0];
+ const postId = segments[2];
+ return {
+ kind: "post",
+ handle,
+ postId,
+ normalizedUrl: `https://x.com/${handle}/status/${postId}`,
+ };
+ }
+
+ if (
+ segments.length === 1 &&
+ isHandle(segments[0]) &&
+ !RESERVED_PROFILE_PATHS.has(segments[0].toLowerCase())
+ ) {
+ const handle = segments[0];
+ return {
+ kind: "profile",
+ handle,
+ normalizedUrl: `https://x.com/${handle}`,
+ };
+ }
+
+ return null;
+}
+
+export function isXTwitterUrl(url: string): boolean {
+ return parseXTwitterUrl(url) !== null;
+}
+
+function isHandle(value: string): boolean {
+ return /^[A-Za-z0-9_]{1,15}$/.test(value);
+}
+
+function isPostId(value: string): boolean {
+ return /^\d{5,}$/.test(value);
+}
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index c06003e2e..900220194 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -9,6 +9,7 @@ import {
} from "./store";
import { judgeCreditsForJudgedCount } from "./search/billing";
import type { MonitorTarget } from "./types";
+import { config } from "../../config";
describe("monitoring store credit helpers", () => {
it("estimates goal-enabled scrape monitors from scrape option costs", () => {
@@ -29,6 +30,108 @@ describe("monitoring store credit helpers", () => {
expect(estimateMonitorCreditsPerRun(targets, true)).toBe(12);
});
+ // The x-twitter engine only exists with an xAI key or DB auth.
+ const withXTwitterEngine = (fn: () => void) => {
+ const saved = config.USE_DB_AUTHENTICATION;
+ config.USE_DB_AUTHENTICATION = true;
+ try {
+ fn();
+ } finally {
+ config.USE_DB_AUTHENTICATION = saved;
+ }
+ };
+
+ it("includes the x-twitter surcharge, so the hold covers what is billed", () =>
+ withXTwitterEngine(() => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-x",
+ type: "scrape",
+ urls: ["https://x.com/blknoiz06"],
+ scrapeOptions: {},
+ },
+ ];
+
+ // 1 scrape + 29 x-twitter, + 1 judge.
+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(30);
+ expect(estimateMonitorCreditsPerRun(targets, true)).toBe(31);
+ expect(
+ calculateMonitorCheckActualCreditsFromPages(
+ [
+ {
+ target_id: "target-x",
+ status: "changed",
+ metadata: {
+ creditsUsed: null,
+ postprocessorsUsed: ["x-twitter"],
+ },
+ judgment: { meaningful: false },
+ },
+ ],
+ targets,
+ ),
+ ).toBe(estimateMonitorCreditsPerRun(targets, true));
+ }));
+
+ it("charges the surcharge only for URLs the x-twitter engine takes", () =>
+ withXTwitterEngine(() => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-mixed",
+ type: "scrape",
+ urls: [
+ "https://twitter.com/someone/status/1234567890",
+ "https://x.com/home",
+ "https://example.com/x.com",
+ ],
+ scrapeOptions: {},
+ },
+ ];
+
+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(3 + 29);
+ }));
+
+ it("skips the surcharge when a browser profile bypasses the x-twitter engine", () =>
+ withXTwitterEngine(() => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-profile",
+ type: "scrape",
+ urls: ["https://x.com/someone"],
+ scrapeOptions: { profile: { name: "p" } },
+ },
+ ];
+
+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(1);
+ }));
+
+ it("skips the surcharge where the x-twitter engine cannot run", () => {
+ const target = (
+ scrapeOptions: Record<string, unknown>,
+ ): MonitorTarget[] => [
+ {
+ id: "t",
+ type: "scrape",
+ urls: ["https://x.com/someone"],
+ scrapeOptions,
+ },
+ ];
+ withXTwitterEngine(() => {
+ // Lockdown serves from the index: 1 base + 4 lockdown, no surcharge.
+ expect(
+ estimateMonitorCreditsPerRun(target({ lockdown: true }), false),
+ ).toBe(5);
+ });
+ const saved = [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] as const;
+ config.USE_DB_AUTHENTICATION = false;
+ config.XAI_API_KEY = undefined;
+ try {
+ expect(estimateMonitorCreditsPerRun(target({}), false)).toBe(1);
+ } finally {
+ [config.USE_DB_AUTHENTICATION, config.XAI_API_KEY] = saved;
+ }
+ });
+
it("adds predictable lockdown costs and judge credits separately", () => {
const targets: MonitorTarget[] = [
{
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 9c9f31211..94ec5a054 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -4,7 +4,9 @@ import { and, asc, count, desc, eq, isNull, ne, sql } from "drizzle-orm";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { monitoringClaimDueMonitors } from "../../db/rpc";
+import { config } from "../../config";
import { shouldParsePDF } from "../../controllers/v2/types";
+import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";
import {
getNextMonitorRunAt,
estimateRunsPerMonth,
@@ -173,13 +175,37 @@ function estimateSearchTargetCredits(
);
}
+/**
+ * The x-twitter engine's surcharge, which billing adds per page (see
+ * fallbackBaseCreditsForPage). Left out of the estimate, a judged one-URL X
+ * monitor reserves 2 credits and costs 31, and an account with a few credits
+ * left passes every hold and is never charged.
+ */
+function xTwitterSurcharge(
+ url: string,
+ options: MonitorTarget["scrapeOptions"],
+): number {
+ // Mirrors the engine router (scrapeURL/engines/index.ts): the engine exists
+ // only with an xAI key or DB auth, lockdown serves from the index alone, and
+ // a browser profile routes around it.
+ const engineEnabled =
+ (config.XAI_API_KEY !== undefined && config.XAI_API_KEY !== "") ||
+ config.USE_DB_AUTHENTICATION === true;
+ if (!engineEnabled || options?.lockdown || options?.profile) return 0;
+ return isXTwitterUrl(url) ? X_TWITTER_POSTPROCESSOR_CREDIT_BONUS : 0;
+}
+
function estimateTargetBaseCredits(
target: MonitorTarget,
judgeEnabled: boolean = false,
): number {
const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);
if (target.type === "scrape") {
- return target.urls.length * creditsPerPage;
+ return target.urls.reduce(
+ (sum, url) =>
+ sum + creditsPerPage + xTwitterSurcharge(url, target.scrapeOptions),
+ 0,
+ );
}
if (target.type === "search") {
return estimateSearchTargetCredits(target, judgeEnabled);