Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
SERP analysis: depth 20 by default, load top 100 on demand; classify DataForSEO timeouts (#523)
Ben Senescu022a7f194479
diff --git a/src/client/features/keywords/components/SerpAnalysisCard.tsx b/src/client/features/keywords/components/SerpAnalysisCard.tsxindex 1df30c21..66b593cd 100644--- a/src/client/features/keywords/components/SerpAnalysisCard.tsx+++ b/src/client/features/keywords/components/SerpAnalysisCard.tsx@@ -6,8 +6,11 @@ export function SerpAnalysisCard({items,keyword,loading,+ loadingMore,+ canLoadMore,error,onRetry,+ deepFetchFailed,page,pageSize,onPageChange,@@ -15,8 +18,14 @@ export function SerpAnalysisCard({items: SerpResultItem[];keyword?: string | null;loading: boolean;+ /** A deeper snapshot is being fetched; `items` is still the shallow one. */+ loadingMore: boolean;+ /** Paging past the loaded results can buy a deeper snapshot. */+ canLoadMore: boolean;error?: string | null;onRetry?: () => void;+ /** The failure was the deeper crawl, so retrying restores the shallow one. */+ deepFetchFailed: boolean;page: number;pageSize: number;onPageChange: (p: number) => void;@@ -31,7 +40,7 @@ export function SerpAnalysisCard({<p>{error}</p>{onRetry ? (<button className="btn btn-xs" onClick={onRetry}>- Retry+ {deepFetchFailed ? "Show top 20" : "Retry"}</button>) : null}</div>@@ -56,10 +65,16 @@ export function SerpAnalysisCard({feature="serp_analysis"/></div>- <SerpAnalysisTable items={pageItems} />+ {pageItems.length === 0 && loadingMore ? (+ <SerpAnalysisLoadingState />+ ) : (+ <SerpAnalysisTable items={pageItems} />+ )}<SerpAnalysisPaginationpage={page}totalPages={totalPages}+ loadingMore={loadingMore}+ canLoadMore={canLoadMore}onPageChange={onPageChange}/></div>@@ -113,23 +128,38 @@ function SerpAnalysisTable({ items }: { items: SerpResultItem[] }) {function SerpAnalysisPagination({page,totalPages,+ loadingMore,+ canLoadMore,onPageChange,}: {page: number;totalPages: number;+ loadingMore: boolean;+ canLoadMore: boolean;onPageChange: (p: number) => void;}) {- if (totalPages <= 1) return null;+ if (totalPages <= 1 && !canLoadMore) return null;++ // Past the loaded results, "Next" stops being free paging and buys a deeper+ // crawl — say so on the button rather than spending silently.+ const nextBuysDeeperSnapshot =+ canLoadMore && !loadingMore && page >= totalPages - 1;return (<div className="flex items-center justify-between mt-3 pt-3 border-t border-base-200"><span className="text-xs text-base-content/50">- Page {page + 1} of {totalPages}+ {loadingMore ? (+ "Loading more results…"+ ) : (+ <>+ Page {page + 1} of {totalPages}+ </>+ )}</span><div className="flex gap-1"><buttonclassName="btn btn-ghost btn-xs"- disabled={page === 0}+ disabled={page === 0 || loadingMore}onClick={() => onPageChange(page - 1)}><ChevronLeft className="size-3.5" />@@ -137,10 +167,10 @@ function SerpAnalysisPagination({</button><buttonclassName="btn btn-ghost btn-xs"- disabled={page >= totalPages - 1}+ disabled={loadingMore || (page >= totalPages - 1 && !canLoadMore)}onClick={() => onPageChange(page + 1)}>- Next+ {nextBuysDeeperSnapshot ? "Load top 100" : "Next"}<ChevronRight className="size-3.5" /></button></div>diff --git a/src/client/features/keywords/hooks/useKeywordSerpAnalysis.ts b/src/client/features/keywords/hooks/useKeywordSerpAnalysis.tsindex 12dd8a63..89c9f715 100644--- a/src/client/features/keywords/hooks/useKeywordSerpAnalysis.ts+++ b/src/client/features/keywords/hooks/useKeywordSerpAnalysis.ts@@ -1,32 +1,67 @@import { useQuery } from "@tanstack/react-query";-import { useState } from "react";+import { useCallback, useState } from "react";import { getStandardErrorMessage } from "@/client/lib/error-messages";import { getSerpAnalysis } from "@/serverFunctions/keywords";+const SERP_PAGE_SIZE = 10;+/** Depth every SERP panel opens at — two pages of results, ~5 credits. */+const SERP_INITIAL_DEPTH = 20;+/**+ * Depth we buy when a user pages past what's loaded. Google has no offset, so+ * this re-crawls the top of the SERP: the deeper snapshot replaces the shallow+ * one instead of extending it, and costs ~20 more credits.+ */+const SERP_DEEP_DEPTH = 100;+export function useKeywordSerpAnalysis(projectId: string,locationCode: number | undefined,) {- const [serpKeyword, setSerpKeyword] = useState<string | null>(null);- const [serpPage, setSerpPage] = useState(0);- const SERP_PAGE_SIZE = 10;+ const [serpKeyword, setSerpKeywordState] = useState<string | null>(null);+ const [serpPage, setSerpPageState] = useState(0);+ const [requestedDepth, setRequestedDepth] = useState<20 | 100>(+ SERP_INITIAL_DEPTH,+ );++ // Everything but the depth identifies the snapshot; the depth decides how+ // deep it goes.+ const snapshotKey = ["serpAnalysis", projectId, serpKeyword, locationCode];const serpQuery = useQuery({- queryKey: ["serpAnalysis", projectId, serpKeyword, locationCode],+ queryKey: [...snapshotKey, requestedDepth],queryFn: () =>getSerpAnalysis({data: {projectId,keyword: serpKeyword!,locationCode,+ depth: requestedDepth,},}),+ // Keep the shallow snapshot on screen while the deeper refetch runs — but+ // only when nothing except the depth changed, so a new keyword or market+ // shows the loading state instead of the previous SERP.+ placeholderData: (previous, previousQuery) => {+ const previousKey = previousQuery?.queryKey;+ const sameSnapshot = snapshotKey.every(+ (part, index) => previousKey?.[index] === part,+ );+ return sameSnapshot ? previous : undefined;+ },+ // Every fetch here is billed, and a deep crawl can outrun DataForSEO's+ // request deadline — a timed-out call may already be billed upstream while+ // metering nothing. Never replay or re-issue one without the user asking;+ // the card offers an explicit retry.+ retry: false,+ refetchOnReconnect: false,+ refetchOnWindowFocus: false,enabled: !!serpKeyword,// Every attempt is a fresh billed DataForSEO task, so a failure must not be// retried automatically — the user re-runs the search instead.retry: false,});+ const { refetch: refetchSerp } = serpQuery;const serpResults = serpQuery.data?.items ?? [];const activeSerpKeyword =serpKeyword ?? serpQuery.data?.requestedKeyword ?? null;@@ -35,16 +70,72 @@ export function useKeywordSerpAnalysis(? getStandardErrorMessage(serpQuery.error, "Failed to load SERP data."): null;+ // A deeper fetch is available until the loaded snapshot is the deep one.+ const canLoadMoreSerp =+ (serpQuery.data?.depth ?? SERP_DEEP_DEPTH) < SERP_DEEP_DEPTH;+ const serpLoadingMore = serpQuery.isPlaceholderData;++ // A failed deep fetch leaves nothing on screen, and retrying it re-buys the+ // expensive call. The card offers the shallow snapshot back instead.+ const deepFetchFailed =+ serpQuery.isError && requestedDepth === SERP_DEEP_DEPTH;++ // Retrying a failed deep crawl re-buys the expensive call that just timed+ // out. Drop back to the shallow snapshot instead: it's still cached, so the+ // user gets their results back for free and can ask for 100 again.+ const retrySerp = useCallback(() => {+ if (requestedDepth === SERP_DEEP_DEPTH) {+ setRequestedDepth(SERP_INITIAL_DEPTH);+ return;+ }+ void refetchSerp();+ }, [refetchSerp, requestedDepth]);++ const setSerpKeyword = useCallback((keyword: string | null) => {+ setSerpKeywordState(keyword);+ setRequestedDepth(SERP_INITIAL_DEPTH);+ }, []);++ const loadedPages = Math.max(+ 1,+ Math.ceil(serpResults.length / SERP_PAGE_SIZE),+ );+ // While the deeper snapshot loads the user stays on the page they asked for;+ // once it lands, a SERP too small to fill that page falls back to the last.+ const visiblePage = serpLoadingMore+ ? serpPage+ : Math.min(serpPage, loadedPages - 1);++ const setSerpPage = useCallback(+ (nextPage: number) => {+ // Paging forward past the loaded results buys the full 100-deep snapshot;+ // the user lands on the requested page once it arrives.+ if (+ nextPage > visiblePage &&+ nextPage >= loadedPages &&+ canLoadMoreSerp+ ) {+ setRequestedDepth(SERP_DEEP_DEPTH);+ }+ setSerpPageState(nextPage);+ },+ [canLoadMoreSerp, loadedPages, visiblePage],+ );+return {serpKeyword,setSerpKeyword,- serpPage,+ retrySerp,+ serpPage: visiblePage,setSerpPage,SERP_PAGE_SIZE,serpQuery,serpResults,activeSerpKeyword,serpLoading,+ serpLoadingMore,+ canLoadMoreSerp,+ deepFetchFailed,serpError,};}diff --git a/src/client/features/keywords/page/KeywordResearchDesktopResults.tsx b/src/client/features/keywords/page/KeywordResearchDesktopResults.tsxindex 1dc7bfb8..b273cb20 100644--- a/src/client/features/keywords/page/KeywordResearchDesktopResults.tsx+++ b/src/client/features/keywords/page/KeywordResearchDesktopResults.tsx@@ -373,8 +373,11 @@ function DesktopSerpPanel({ controller }: Props) {items={controller.serpResults}keyword={controller.activeSerpKeyword}loading={controller.serpLoading}+ loadingMore={controller.serpLoadingMore}+ canLoadMore={controller.canLoadMoreSerp}error={controller.serpError}- onRetry={() => void controller.serpQuery.refetch()}+ onRetry={controller.retrySerp}+ deepFetchFailed={controller.deepFetchFailed}page={controller.serpPage}pageSize={controller.SERP_PAGE_SIZE}onPageChange={controller.setSerpPage}diff --git a/src/client/features/keywords/page/KeywordResearchMobileResults.tsx b/src/client/features/keywords/page/KeywordResearchMobileResults.tsxindex 56ea1b10..489469f2 100644--- a/src/client/features/keywords/page/KeywordResearchMobileResults.tsx+++ b/src/client/features/keywords/page/KeywordResearchMobileResults.tsx@@ -68,8 +68,11 @@ export function KeywordResearchMobileResults({ controller }: Props) {items={controller.serpResults}keyword={controller.activeSerpKeyword}loading={controller.serpLoading}+ loadingMore={controller.serpLoadingMore}+ canLoadMore={controller.canLoadMoreSerp}error={controller.serpError}- onRetry={() => void controller.serpQuery.refetch()}+ onRetry={controller.retrySerp}+ deepFetchFailed={controller.deepFetchFailed}page={controller.serpPage}pageSize={controller.SERP_PAGE_SIZE}onPageChange={controller.setSerpPage}diff --git a/src/client/features/keywords/state/useKeywordResearchController.ts b/src/client/features/keywords/state/useKeywordResearchController.tsindex 7a3ed5ab..392d8fa9 100644--- a/src/client/features/keywords/state/useKeywordResearchController.ts+++ b/src/client/features/keywords/state/useKeywordResearchController.ts@@ -76,6 +76,10 @@ export function useKeywordResearchController(serpResults,activeSerpKeyword,serpLoading,+ serpLoadingMore,+ canLoadMoreSerp,+ deepFetchFailed,+ retrySerp,serpError,} = useKeywordSerpAnalysis(input.projectId, locationCode);@@ -260,11 +264,15 @@ export function useKeywordResearchController(researchMutationError,retrySearch,resetFilters,+ retrySerp,rows,searchedKeyword,selectedRows,+ canLoadMoreSerp,+ deepFetchFailed,serpError,serpLoading,+ serpLoadingMore,serpPage,serpQuery,serpResults,diff --git a/src/server/features/keywords/services/research/serp.test.ts b/src/server/features/keywords/services/research/serp.test.tsnew file mode 100644index 00000000..cc7d6ddf--- /dev/null+++ b/src/server/features/keywords/services/research/serp.test.ts@@ -0,0 +1,124 @@+import { describe, expect, it, vi } from "vitest";+import type { BillingCustomerContext } from "@/server/billing/subscription";++const mocks = vi.hoisted(() => ({+ createDataforseoClient: vi.fn(),+ getCached: vi.fn(),+ setCached: vi.fn(async () => {}),+}));++vi.mock("cloudflare:workers", () => ({ waitUntil: vi.fn() }));++vi.mock("@/server/lib/r2-cache", () => ({+ buildCacheKey: vi.fn(async () => "serp:analysis:key"),+ getCached: mocks.getCached,+ setCached: mocks.setCached,+}));++vi.mock("@/server/lib/dataforseo", () => ({+ SERP_ANALYSIS_DEPTH: 20,+ createDataforseoClient: mocks.createDataforseoClient,+}));++import { getSerpAnalysis } from "./serp";++const billingCustomer: BillingCustomerContext = {+ organizationId: "org_1",+ userId: "user_1",+ userEmail: "[email protected]",+};++const input = {+ projectId: "proj_1",+ keyword: "seo tools",+ locationCode: 2840,+ languageCode: "en",+};++function cachedSnapshot(depth: number) {+ return {+ requestedKeyword: "seo tools",+ depth,+ items: [+ {+ rank: 1,+ title: "Cached",+ url: "https://cached.example",+ domain: "cached.example",+ description: "",+ etv: null,+ estimatedPaidTrafficCost: null,+ referringDomains: null,+ backlinks: null,+ isNew: false,+ rankChange: null,+ },+ ],+ };+}++function mockLiveSerp() {+ const live = vi.fn().mockResolvedValue([+ {+ type: "organic",+ rank_group: 1,+ title: "Live",+ url: "https://live.example",+ domain: "live.example",+ },+ ]);+ mocks.createDataforseoClient.mockReturnValue({ serp: { live } });+ return live;+}++describe("getSerpAnalysis cache depth", () => {+ it("keeps a good snapshot when the deeper crawl comes back empty", async () => {+ mocks.getCached.mockResolvedValue(cachedSnapshot(20));+ mocks.createDataforseoClient.mockReturnValue({+ serp: { live: vi.fn().mockResolvedValue([]) },+ });++ const result = await getSerpAnalysis(+ { ...input, depth: 100 },+ billingCustomer,+ );++ expect(result.items).toEqual([]);+ // Caching the empty deep result would answer every shallower request with+ // nothing until the entry expires.+ expect(mocks.setCached).not.toHaveBeenCalled();+ });++ it("serves a depth-100 snapshot for a depth-20 request", async () => {+ mocks.getCached.mockResolvedValue(cachedSnapshot(100));+ const live = mockLiveSerp();++ const result = await getSerpAnalysis(+ { ...input, depth: 20 },+ billingCustomer,+ );++ expect(live).not.toHaveBeenCalled();+ expect(result.depth).toBe(100);+ expect(result.items[0]?.title).toBe("Cached");+ });++ it("refetches live at depth 100 when only a depth-20 snapshot is cached", async () => {+ mocks.getCached.mockResolvedValue(cachedSnapshot(20));+ const live = mockLiveSerp();++ const result = await getSerpAnalysis(+ { ...input, depth: 100 },+ billingCustomer,+ );++ expect(live).toHaveBeenCalledWith(expect.objectContaining({ depth: 100 }));+ expect(result.items[0]?.title).toBe("Live");+ // The deeper snapshot replaces the shallow entry under the same key.+ expect(mocks.setCached).toHaveBeenCalledWith(+ "serp:analysis:key",+ expect.objectContaining({ depth: 100 }),+ expect.any(Number),+ );+ });+});diff --git a/src/server/features/keywords/services/research/serp.ts b/src/server/features/keywords/services/research/serp.tsindex 8eb404f7..96bbe8a5 100644--- a/src/server/features/keywords/services/research/serp.ts+++ b/src/server/features/keywords/services/research/serp.ts@@ -1,5 +1,8 @@import { waitUntil } from "cloudflare:workers";-import { type SerpLiveItem } from "@/server/lib/dataforseo";+import {+ SERP_ANALYSIS_DEPTH,+ type SerpLiveItem,+} from "@/server/lib/dataforseo";import { buildCacheKey, getCached, setCached } from "@/server/lib/r2-cache";import type { SerpResultItem } from "@/types/keywords";import { z } from "zod";@@ -14,6 +17,8 @@ type SerpAnalysisReason = "no_organic_results";type SerpAnalysisResult = {requestedKeyword: string;items: SerpResultItem[];+ /** SERP depth this snapshot was crawled at — how deep the results go. */+ depth: number;reason?: SerpAnalysisReason;};@@ -34,6 +39,10 @@ const serpResultItemSchema = z.object({const serpCacheSchema = z.object({requestedKeyword: z.string(),items: z.array(serpResultItemSchema),+ // Entries written before depth was tracked don't say how deep they went, so+ // they're treated as the shallow floor: worst case a user re-buys a deeper+ // crawl once, rather than being shown 20 results as if they were 100.+ depth: z.number().int().default(SERP_ANALYSIS_DEPTH),reason: z.enum(["no_organic_results"]).optional(),});@@ -61,10 +70,12 @@ async function getSerpLiveAnalysis(keyword: string;locationCode: number;languageCode: string;+ depth: number;},billingCustomer: BillingCustomerContext,): Promise<SerpAnalysisResult> {const keyword = normalizeKeyword(input.keyword);+ const { depth } = input;const cacheKey = await buildCacheKey("serp:analysis", {organizationId: billingCustomer.organizationId,@@ -74,9 +85,13 @@ async function getSerpLiveAnalysis(languageCode: input.languageCode,});+ // Depth lives in the cached value, not the key: DataForSEO has no offset, so+ // a deeper crawl re-fetches the head too. A depth-100 snapshot therefore also+ // answers depth-20 requests, and a deeper request replaces the entry rather+ // than appending to it.const cachedRaw = await getCached(cacheKey);const cached = serpCacheSchema.safeParse(cachedRaw);- if (cached.success) {+ if (cached.success && cached.data.depth >= depth) {return cached.data;}@@ -84,14 +99,27 @@ async function getSerpLiveAnalysis(keyword,locationCode: input.locationCode,languageCode: input.languageCode,+ depth,});const items = mapOrganicSerpItems(liveItems);- const result: SerpAnalysisResult = { requestedKeyword: keyword, items };+ const result: SerpAnalysisResult = {+ requestedKeyword: keyword,+ items,+ depth,+ };if (items.length === 0) {result.reason = "no_organic_results";}+ // DataForSEO reports "no results" transiently, and a deeper crawl overwrites+ // the same key. Without this guard one empty re-crawl would evict a good+ // snapshot and, being the deeper entry, answer every shallower request with+ // nothing for the rest of the TTL.+ if (items.length === 0 && cached.success && cached.data.items.length > 0) {+ return result;+ }+// waitUntil, not void: workerd cancels unregistered pending I/O once the// response is sent, so a fire-and-forget put never persists the cache.waitUntil(diff --git a/src/server/lib/dataforseo/core.test.ts b/src/server/lib/dataforseo/core.test.tsindex 5d231a13..cb5ff7e3 100644--- a/src/server/lib/dataforseo/core.test.ts+++ b/src/server/lib/dataforseo/core.test.ts@@ -29,4 +29,26 @@ describe("DataForSEO transport", () => {"Basic encoded-credentials",);});++ // The request-deadline abort arrives as a bare DOMException. Left unclassified+ // it escapes as an anonymous INTERNAL_ERROR; retrying it would replay a call+ // DataForSEO may already have billed. Both names are reachable: the shared+ // budget aborts with TimeoutError, Lighthouse's own controller with AbortError.+ it.each(["TimeoutError", "AbortError"])(+ "maps a %s abort to UPSTREAM_UNAVAILABLE without retrying",+ async (name) => {+ const fetchMock = vi+ .fn<typeof fetch>()+ .mockRejectedValue(new DOMException("aborted", name));+ vi.stubGlobal("fetch", fetchMock);++ await expect(+ dataforseoPost("/v3/serp/google/organic/live/advanced", []),+ ).rejects.toMatchObject({+ code: "UPSTREAM_UNAVAILABLE",+ name: "DataForSEOTimeoutError",+ });+ expect(fetchMock).toHaveBeenCalledOnce();+ },+ );});diff --git a/src/server/lib/dataforseo/core.ts b/src/server/lib/dataforseo/core.tsindex 87d68ce2..fe62c687 100644--- a/src/server/lib/dataforseo/core.ts+++ b/src/server/lib/dataforseo/core.ts@@ -75,7 +75,34 @@ function createAuthenticatedFetch(init?.signal ?? AbortSignal.timeout(DATAFORSEO_REQUEST_TIMEOUT_MS);for (let attempt = 0; ; attempt++) {- const response = await fetch(url, { ...init, headers, signal });+ let response: Response;+ try {+ response = await fetch(url, { ...init, headers, signal });+ } catch (error) {+ // Both abort flavours mean "we ran out of time", and neither carries a+ // useful name of its own: the shared budget above rejects with+ // TimeoutError, while Lighthouse passes its own AbortController+ // deadline via init.signal and rejects with AbortError. Classify them+ // so callers get a provider-degradation error instead of an anonymous+ // internal one. Deliberately not retried: a call that ran past the+ // deadline may already be billed by DataForSEO, and because this is not+ // a DataforseoChargedTaskError the customer is metered nothing for it,+ // so replaying would be spend we eat twice.+ if (+ error instanceof Error &&+ (error.name === "TimeoutError" || error.name === "AbortError")+ ) {+ const path = formatDataforseoRequestPath(url);+ const timeoutError = new AppError(+ "UPSTREAM_UNAVAILABLE",+ `DataForSEO request timed out on ${path}`,+ { provider: "dataforseo", providerPath: path },+ );+ timeoutError.name = "DataForSEOTimeoutError";+ throw timeoutError;+ }+ throw error;+ }if (response.ok) return response;// Transient upstream 5xx on an idempotent read -> back off and retry.diff --git a/src/server/lib/dataforseo/index.ts b/src/server/lib/dataforseo/index.tsindex 2123ad09..e9b652c7 100644--- a/src/server/lib/dataforseo/index.ts+++ b/src/server/lib/dataforseo/index.ts@@ -18,6 +18,8 @@ export {type LlmPlatform,} from "@/server/lib/dataforseo/shared";+export { SERP_ANALYSIS_DEPTH } from "@/server/lib/dataforseo/serp";+export { normalizeBacklinksTarget } from "@/server/lib/dataforseoBacklinksTarget";// Section fetchers called outside the metered client. Task collection is freediff --git a/src/server/lib/dataforseo/serp.ts b/src/server/lib/dataforseo/serp.tsindex 50b788cf..850601fc 100644--- a/src/server/lib/dataforseo/serp.ts+++ b/src/server/lib/dataforseo/serp.ts@@ -13,6 +13,15 @@ import {} from "@/server/lib/dataforseo/envelope";import { AppError } from "@/server/lib/errors";+// Default depth for keyword SERP analysis. DataForSEO crawls (and bills) one+// Google page of 10 results at a time, and the crawls are sequential, so depth+// is the single lever on both latency and cost here: every 10 results is+// another page fetch against the shared 60s request budget. Keep this low —+// callers that need to see deeper ranks pass an explicit depth. There is no+// offset/cursor: a deeper request re-crawls pages 1..N/10 from the top, so it+// replaces the shallow snapshot rather than extending it.+export const SERP_ANALYSIS_DEPTH = 20;+/** DataForSEO bills SERPs in pages of 10; depth outside 10-100 is rejected. */function clampSerpDepth(depth: number): number {return Math.min(100, Math.max(10, depth));@@ -78,6 +87,7 @@ export async function fetchLiveSerp(input: {keyword: string;locationCode: number;languageCode: string;+ depth?: number;}): Promise<DataforseoApiResponse<SerpLiveItem[]>> {const response = await dataforseoPost("/v3/serp/google/organic/live/advanced",@@ -88,7 +98,7 @@ export async function fetchLiveSerp(input: {language_code: input.languageCode,device: "desktop",os: "windows",- depth: 100,+ depth: clampSerpDepth(input.depth ?? SERP_ANALYSIS_DEPTH),},],);diff --git a/src/server/mcp/tools/get-serp-results.ts b/src/server/mcp/tools/get-serp-results.tsindex 0843c6d5..70e140e1 100644--- a/src/server/mcp/tools/get-serp-results.ts+++ b/src/server/mcp/tools/get-serp-results.ts@@ -1,5 +1,8 @@import { z } from "zod";-import { createDataforseoClient } from "@/server/lib/dataforseo";+import {+ createDataforseoClient,+ SERP_ANALYSIS_DEPTH,+} from "@/server/lib/dataforseo";import { mcpResponse } from "@/server/mcp/formatters";import { buildProjectMeta } from "@/server/mcp/context";import { optionalMetaOutputSchema } from "@/server/mcp/output-schemas";@@ -43,6 +46,16 @@ const inputSchema = {.describe("1-10 queries. Bulk-friendly — prefer this over multiple single-query calls.",),+ depth: z+ .number()+ .int()+ .min(10)+ .max(100)+ .multipleOf(10)+ .optional()+ .describe(+ "How many SERP rows to crawl per keyword — a multiple of 10 from 10 to 100, default 20. Google has no offset, so a deeper crawl re-fetches the top too: each additional 10 adds ~2.5 credits per keyword. Only raise it when you need ranks past the top 20.",+ ),} as const;type Args = z.infer<z.ZodObject<typeof inputSchema>>;@@ -52,7 +65,7 @@ export const getSerpResultsTool = {config: {title: "Get Google SERP results",description:- "Fetch live Google organic search results for 1-10 keywords. Use this to inspect who ranks for a query, verify competitors, compare SERPs across keywords, or gather source URLs before content planning. Charges credits per keyword (~30-60 each). Does not save results to OpenSEO. Per-keyword errors don't fail the batch.",+ "Fetch live Google organic search results for 1-10 keywords. Use this to inspect who ranks for a query, verify competitors, compare SERPs across keywords, or gather source URLs before content planning. Returns the top `depth` result rows per keyword (default 20). Charges credits per keyword: ~5 each at the default depth 20, and each additional 10 of depth adds ~2.5. Does not save results to OpenSEO. Per-keyword errors don't fail the batch.",inputSchema,outputSchema: {results: z.array(@@ -94,15 +107,17 @@ export const getSerpResultsTool = {},handler: withMcpProjectAuth(async (args: Args, context) => {const client = createDataforseoClient(context.billing);+ const depth = args.depth ?? SERP_ANALYSIS_DEPTH;const results = await Promise.all(args.queries.map(async (q) => {try {const items = await client.serp.live({keyword: q.keyword,...resolveMarket(q, context.project),+ depth,});// Trim noise — return only essentials per item.- const trimmed = items.slice(0, 20).map((item) => ({+ const trimmed = items.slice(0, depth).map((item) => ({type: item.type,rank: item.rank_absolute ?? item.rank_group ?? null,title: item.title ?? null,diff --git a/src/server/mcp/tools/tool-text-output.test.ts b/src/server/mcp/tools/tool-text-output.test.tsindex d2c94587..1971ed90 100644--- a/src/server/mcp/tools/tool-text-output.test.ts+++ b/src/server/mcp/tools/tool-text-output.test.ts@@ -42,6 +42,7 @@ vi.mock("@/server/lib/dataforseo", async () => {createDataforseoClient: mocks.createDataforseoClient,fetchBusinessDataTaskResult: mocks.fetchBusinessDataTaskResult,normalizeBacklinksTarget: targets.normalizeBacklinksTarget,+ SERP_ANALYSIS_DEPTH: 20,};});vi.mock("@/server/features/projects/services/ProjectService", () => ({@@ -387,4 +388,31 @@ describe("MCP tool text output (service-backed tools)", () => {"1 | example.com | Best SEO Tools | https://example.com/best",);});++ it("get_serp_results crawls and returns rows to the requested depth", async () => {+ const live = vi.fn().mockResolvedValue(+ Array.from({ length: 40 }, (_, index) => ({+ type: "organic",+ rank_absolute: index + 1,+ title: `Result ${index + 1}`,+ url: `https://example.com/${index + 1}`,+ domain: "example.com",+ description: "desc",+ })),+ );+ mocks.createDataforseoClient.mockReturnValue({ serp: { live } });++ const result = await getSerpResultsTool.handler(+ {+ projectId: "project_1",+ queries: [{ keyword: "seo tools" }],+ depth: 30,+ },+ toolContext,+ );++ expect(live).toHaveBeenCalledWith(expect.objectContaining({ depth: 30 }));+ // Rows are trimmed to the depth that was crawled, not the fixed top 20.+ expect(textContent(result)).toContain('"seo tools" (30 results)');+ });});diff --git a/src/types/schemas/keywords.ts b/src/types/schemas/keywords.tsindex 56ec5fa0..b1ccfea6 100644--- a/src/types/schemas/keywords.ts+++ b/src/types/schemas/keywords.ts@@ -176,6 +176,10 @@ export const serpAnalysisSchema = z.object({keyword: z.string().min(1),locationCode: z.number().int().positive().optional(),languageCode: z.string().min(2).max(8).optional(),+ // Only the two depths the app offers: the default top-20 snapshot, and the+ // full 100 the SERP panel buys when a user pages past the loaded results.+ // Each 10 of depth is another crawled Google page (~2.5 credits).+ depth: z.union([z.literal(20), z.literal(100)]).default(20),});/* ------------------------------------------------------------------ */