Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Onboarding chat: let read_website fetch specific user-named URLs (#300)
Ben Senescu616b6727fa45
diff --git a/src/server/features/onboarding/OnboardingChatAgent.ts b/src/server/features/onboarding/OnboardingChatAgent.tsindex 0d186a1a..4dbb141c 100644--- a/src/server/features/onboarding/OnboardingChatAgent.ts+++ b/src/server/features/onboarding/OnboardingChatAgent.ts@@ -48,7 +48,7 @@ function buildSystemPrompt(domain: string | null): string {"When a request is beyond your preview tools, don't conclude OpenSEO can't do it — describe what the full product does per the fact sheet, and don't claim capabilities the fact sheet doesn't list.","You have tools to pull real search data. Never state a metric, search volume, keyword difficulty, ranking, or competitor figure you did not get from a tool.","Core tools for THIS user's own site — use these freely whenever the user asks you to analyze their site, recommend a strategy, or for any site-specific advice:",- "- read_website: reads their pages as plain text. Always available.",+ "- read_website: reads web pages as plain text. With no arguments it reads the user's own site; when the user names or pastes specific page URLs (their own pages or a competitor's), pass those as `urls` to read exactly those pages. Always available, no credits — use it whenever the user points you at specific URLs.","- get_seo_metrics: their estimated organic traffic, ranking-keyword count, and the keywords they already rank for (each with real search volume and difficulty). May report it's unavailable for brand-new sites or unsupported markets.","- research_keywords: given one seed topic from their site, returns related keywords each with real monthly search volume and difficulty (KD). Use it to ground keyword suggestions in real data — especially when get_seo_metrics shows no rankings. Seed it with the site's primary topic; call it again only for a clearly distinct second theme.","Market & competitor tools — these cost more credits, so use them SPARINGLY and only when the user's question is specifically about competitors, the live SERP, or backlinks. Do NOT call them just to enrich a routine strategy, and never call more than one or two per reply. The core site tools above answer most questions on their own.",diff --git a/src/server/features/onboarding/onboardingChatTools.ts b/src/server/features/onboarding/onboardingChatTools.tsindex 205ab0a3..5a20860f 100644--- a/src/server/features/onboarding/onboardingChatTools.ts+++ b/src/server/features/onboarding/onboardingChatTools.ts@@ -1,7 +1,11 @@import { tool, type ToolSet } from "ai";import { z } from "zod";import { AppError } from "@/server/lib/errors";-import { readSite } from "@/server/features/onboarding/scrape";+import {+ MAX_PAGES,+ readPages,+ readSite,+} from "@/server/features/onboarding/scrape";import { DomainService } from "@/server/features/domain/services/DomainService";import { KeywordResearchService } from "@/server/features/keywords/services/KeywordResearchService";import { createDataforseoClient } from "@/server/lib/dataforseo";@@ -75,18 +79,34 @@ function coreSiteTools(ctx: ToolContext): ToolSet {return {read_website: tool({description:- "Read the user's own website (their pages, as plain text) to ground site-specific advice and strategy. Uses the project's saved domain.",- inputSchema: z.object({}),- execute: async () => {- if (!project.domain) {- throw new AppError("VALIDATION_ERROR", "Set a website domain first");+ "Read web pages as plain text to ground advice. With no arguments, reads the user's own site (homepage plus a few pages from its sitemap) from the project's saved domain. Pass `urls` to read specific pages the user names instead — particular pages of their site, or a competitor's page to compare. Always available; uses no credits.",+ inputSchema: z.object({+ urls: z+ .array(z.string().url())+ .max(MAX_PAGES)+ .optional()+ .describe(+ `Specific page URLs to read (max ${MAX_PAGES}). Omit to read the user's own site from its saved domain.`,+ ),+ }),+ execute: async ({ urls }) => {+ const site =+ urls && urls.length > 0+ ? await readPages(urls)+ : project.domain+ ? await readSite(project.domain)+ : null;+ if (!site) {+ throw new AppError(+ "VALIDATION_ERROR",+ "Provide page URLs to read, or set a website domain first.",+ );}- const site = await readSite(project.domain);if (site.blocked) {return {blocked: true,pages: [],- note: "Could not read the site's pages. Ask the user to describe what they do, and keep the advice high-level.",+ note: "Could not read the requested page(s). Ask the user to describe what they cover, and keep the advice high-level.",};}return {diff --git a/src/server/features/onboarding/scrape.test.ts b/src/server/features/onboarding/scrape.test.tsindex 0cb3e3bd..68782b5e 100644--- a/src/server/features/onboarding/scrape.test.ts+++ b/src/server/features/onboarding/scrape.test.ts@@ -1,5 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";-import { readSite } from "@/server/features/onboarding/scrape";+import { readPages, readSite } from "@/server/features/onboarding/scrape";describe("readSite SSRF guard", () => {beforeEach(() => {@@ -26,3 +26,33 @@ describe("readSite SSRF guard", () => {expect(fetch).not.toHaveBeenCalled();});});++describe("readPages SSRF guard", () => {+ beforeEach(() => {+ vi.stubGlobal("fetch", vi.fn());+ });++ afterEach(() => {+ vi.unstubAllGlobals();+ });++ it("skips private/metadata URLs without fetching them", async () => {+ const result = await readPages([+ "http://169.254.169.254/latest/meta-data/",+ "http://localhost:3000/admin",+ ]);++ expect(result.blocked).toBe(true);+ expect(result.pages).toEqual([]);+ // Every URL is validated before any outbound fetch.+ expect(fetch).not.toHaveBeenCalled();+ });++ it("returns blocked for an empty URL list without fetching", async () => {+ const result = await readPages([]);++ expect(result.blocked).toBe(true);+ expect(result.pages).toEqual([]);+ expect(fetch).not.toHaveBeenCalled();+ });+});diff --git a/src/server/features/onboarding/scrape.ts b/src/server/features/onboarding/scrape.tsindex 1a637b24..7b389c1f 100644--- a/src/server/features/onboarding/scrape.ts+++ b/src/server/features/onboarding/scrape.ts@@ -6,7 +6,7 @@import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";-const MAX_PAGES = 5;+export const MAX_PAGES = 5;const PER_PAGE_CHAR_LIMIT = 4000;const FETCH_TIMEOUT_MS = 10_000;const MAX_RESPONSE_BYTES = 2_000_000;@@ -19,7 +19,6 @@ type ScrapedPage = {};type SiteReadResult = {- rootUrl: string;pages: ScrapedPage[];/** True when we couldn't read any page (blocked, offline, etc.). */blocked: boolean;@@ -139,37 +138,61 @@ function decodeEntities(value: string): string {.replace(/ /g, " ");}-/** Discovers and reads up to MAX_PAGES pages of a site as plain text. */+/** Fetches one (already-validated) URL and shapes it as a page, or null if it+ * couldn't be read or yielded no text. */+async function scrapePage(url: string): Promise<ScrapedPage | null> {+ const html = await fetchText(url);+ if (!html) {+ return null;+ }+ const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT);+ if (text.length === 0) {+ return null;+ }+ return { url, title: extractTitle(html), text };+}++/**+ * Reads a specific list of page URLs as plain text — used when the user names+ * exact pages (their own or a competitor's) rather than asking us to discover a+ * site. Each URL is independently run through the SSRF guard, so a blocked or+ * unreachable URL is skipped rather than failing the batch.+ */+export async function readPages(urls: string[]): Promise<SiteReadResult> {+ const pages: ScrapedPage[] = [];+ for (const rawUrl of urls.slice(0, MAX_PAGES)) {+ let url: string;+ try {+ // Re-validates host, blocks private/metadata IPs, does DoH DNS resolution.+ url = await normalizeAndValidateStartUrl(rawUrl);+ } catch {+ continue; // blocked or unparseable URL+ }+ const page = await scrapePage(url);+ if (page) {+ pages.push(page);+ }+ }++ return { pages, blocked: pages.length === 0 };+}++/**+ * Discovers a site's representative URLs (homepage + sitemap) and reads them.+ * Just URL discovery on top of readPages, which does the validated fetching.+ */export async function readSite(domain: string): Promise<SiteReadResult> {let rootUrl: string;try {rootUrl = await normalizeAndValidateStartUrl(domain);} catch {// Blocked (private/metadata host) or unparseable domain — nothing to read.- return { rootUrl: `https://${domain}`, pages: [], blocked: true };+ return { pages: [], blocked: true };}const origin = new URL(rootUrl).origin;// Prefer the sitemap for representative URLs; always include the homepage.const sitemap = await fetchText(`${origin}/sitemap.xml`);const discovered = sitemap ? parseSitemapUrls(sitemap, origin) : [];- const targets = [- rootUrl,- ...discovered.filter((url) => url !== rootUrl),- ].slice(0, MAX_PAGES);-- const pages: ScrapedPage[] = [];- for (const url of targets) {- const html = await fetchText(url);- if (!html) {- continue;- }- const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT);- if (text.length === 0) {- continue;- }- pages.push({ url, title: extractTitle(html), text });- }-- return { rootUrl, pages, blocked: pages.length === 0 };+ return readPages([rootUrl, ...discovered.filter((url) => url !== rootUrl)]);}