Skip to content

Open source alternative to Semrush and Ahrefs

Onboarding chat: let read_website fetch specific user-named URLs (#300)

Ben Senescu616b6727fa45
diff --git a/src/server/features/onboarding/OnboardingChatAgent.ts b/src/server/features/onboarding/OnboardingChatAgent.ts
index 0d186a1a..4dbb141c 100644
--- a/src/server/features/onboarding/OnboardingChatAgent.ts
+++ b/src/server/features/onboarding/OnboardingChatAgent.ts
@@ -48,7 +48,7 @@ function buildSystemPrompt(domain: string | null): string {
"When a request is beyond your preview tools, don't conclude OpenSEO can't do it — describe what the full product does per the fact sheet, and don't claim capabilities the fact sheet doesn't list.",
"You have tools to pull real search data. Never state a metric, search volume, keyword difficulty, ranking, or competitor figure you did not get from a tool.",
"Core tools for THIS user's own site — use these freely whenever the user asks you to analyze their site, recommend a strategy, or for any site-specific advice:",
- "- read_website: reads their pages as plain text. Always available.",
+ "- read_website: reads web pages as plain text. With no arguments it reads the user's own site; when the user names or pastes specific page URLs (their own pages or a competitor's), pass those as `urls` to read exactly those pages. Always available, no credits — use it whenever the user points you at specific URLs.",
"- get_seo_metrics: their estimated organic traffic, ranking-keyword count, and the keywords they already rank for (each with real search volume and difficulty). May report it's unavailable for brand-new sites or unsupported markets.",
"- research_keywords: given one seed topic from their site, returns related keywords each with real monthly search volume and difficulty (KD). Use it to ground keyword suggestions in real data — especially when get_seo_metrics shows no rankings. Seed it with the site's primary topic; call it again only for a clearly distinct second theme.",
"Market & competitor tools — these cost more credits, so use them SPARINGLY and only when the user's question is specifically about competitors, the live SERP, or backlinks. Do NOT call them just to enrich a routine strategy, and never call more than one or two per reply. The core site tools above answer most questions on their own.",
diff --git a/src/server/features/onboarding/onboardingChatTools.ts b/src/server/features/onboarding/onboardingChatTools.ts
index 205ab0a3..5a20860f 100644
--- a/src/server/features/onboarding/onboardingChatTools.ts
+++ b/src/server/features/onboarding/onboardingChatTools.ts
@@ -1,7 +1,11 @@
import { tool, type ToolSet } from "ai";
import { z } from "zod";
import { AppError } from "@/server/lib/errors";
-import { readSite } from "@/server/features/onboarding/scrape";
+import {
+ MAX_PAGES,
+ readPages,
+ readSite,
+} from "@/server/features/onboarding/scrape";
import { DomainService } from "@/server/features/domain/services/DomainService";
import { KeywordResearchService } from "@/server/features/keywords/services/KeywordResearchService";
import { createDataforseoClient } from "@/server/lib/dataforseo";
@@ -75,18 +79,34 @@ function coreSiteTools(ctx: ToolContext): ToolSet {
return {
read_website: tool({
description:
- "Read the user's own website (their pages, as plain text) to ground site-specific advice and strategy. Uses the project's saved domain.",
- inputSchema: z.object({}),
- execute: async () => {
- if (!project.domain) {
- throw new AppError("VALIDATION_ERROR", "Set a website domain first");
+ "Read web pages as plain text to ground advice. With no arguments, reads the user's own site (homepage plus a few pages from its sitemap) from the project's saved domain. Pass `urls` to read specific pages the user names instead — particular pages of their site, or a competitor's page to compare. Always available; uses no credits.",
+ inputSchema: z.object({
+ urls: z
+ .array(z.string().url())
+ .max(MAX_PAGES)
+ .optional()
+ .describe(
+ `Specific page URLs to read (max ${MAX_PAGES}). Omit to read the user's own site from its saved domain.`,
+ ),
+ }),
+ execute: async ({ urls }) => {
+ const site =
+ urls && urls.length > 0
+ ? await readPages(urls)
+ : project.domain
+ ? await readSite(project.domain)
+ : null;
+ if (!site) {
+ throw new AppError(
+ "VALIDATION_ERROR",
+ "Provide page URLs to read, or set a website domain first.",
+ );
}
- const site = await readSite(project.domain);
if (site.blocked) {
return {
blocked: true,
pages: [],
- note: "Could not read the site's pages. Ask the user to describe what they do, and keep the advice high-level.",
+ note: "Could not read the requested page(s). Ask the user to describe what they cover, and keep the advice high-level.",
};
}
return {
diff --git a/src/server/features/onboarding/scrape.test.ts b/src/server/features/onboarding/scrape.test.ts
index 0cb3e3bd..68782b5e 100644
--- a/src/server/features/onboarding/scrape.test.ts
+++ b/src/server/features/onboarding/scrape.test.ts
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { readSite } from "@/server/features/onboarding/scrape";
+import { readPages, readSite } from "@/server/features/onboarding/scrape";
describe("readSite SSRF guard", () => {
beforeEach(() => {
@@ -26,3 +26,33 @@ describe("readSite SSRF guard", () => {
expect(fetch).not.toHaveBeenCalled();
});
});
+
+describe("readPages SSRF guard", () => {
+ beforeEach(() => {
+ vi.stubGlobal("fetch", vi.fn());
+ });
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it("skips private/metadata URLs without fetching them", async () => {
+ const result = await readPages([
+ "http://169.254.169.254/latest/meta-data/",
+ "http://localhost:3000/admin",
+ ]);
+
+ expect(result.blocked).toBe(true);
+ expect(result.pages).toEqual([]);
+ // Every URL is validated before any outbound fetch.
+ expect(fetch).not.toHaveBeenCalled();
+ });
+
+ it("returns blocked for an empty URL list without fetching", async () => {
+ const result = await readPages([]);
+
+ expect(result.blocked).toBe(true);
+ expect(result.pages).toEqual([]);
+ expect(fetch).not.toHaveBeenCalled();
+ });
+});
diff --git a/src/server/features/onboarding/scrape.ts b/src/server/features/onboarding/scrape.ts
index 1a637b24..7b389c1f 100644
--- a/src/server/features/onboarding/scrape.ts
+++ b/src/server/features/onboarding/scrape.ts
@@ -6,7 +6,7 @@
import { normalizeAndValidateStartUrl } from "@/server/lib/audit/url-policy";
-const MAX_PAGES = 5;
+export const MAX_PAGES = 5;
const PER_PAGE_CHAR_LIMIT = 4000;
const FETCH_TIMEOUT_MS = 10_000;
const MAX_RESPONSE_BYTES = 2_000_000;
@@ -19,7 +19,6 @@ type ScrapedPage = {
};
type SiteReadResult = {
- rootUrl: string;
pages: ScrapedPage[];
/** True when we couldn't read any page (blocked, offline, etc.). */
blocked: boolean;
@@ -139,37 +138,61 @@ function decodeEntities(value: string): string {
.replace(/ /g, " ");
}
-/** Discovers and reads up to MAX_PAGES pages of a site as plain text. */
+/** Fetches one (already-validated) URL and shapes it as a page, or null if it
+ * couldn't be read or yielded no text. */
+async function scrapePage(url: string): Promise<ScrapedPage | null> {
+ const html = await fetchText(url);
+ if (!html) {
+ return null;
+ }
+ const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT);
+ if (text.length === 0) {
+ return null;
+ }
+ return { url, title: extractTitle(html), text };
+}
+
+/**
+ * Reads a specific list of page URLs as plain text — used when the user names
+ * exact pages (their own or a competitor's) rather than asking us to discover a
+ * site. Each URL is independently run through the SSRF guard, so a blocked or
+ * unreachable URL is skipped rather than failing the batch.
+ */
+export async function readPages(urls: string[]): Promise<SiteReadResult> {
+ const pages: ScrapedPage[] = [];
+ for (const rawUrl of urls.slice(0, MAX_PAGES)) {
+ let url: string;
+ try {
+ // Re-validates host, blocks private/metadata IPs, does DoH DNS resolution.
+ url = await normalizeAndValidateStartUrl(rawUrl);
+ } catch {
+ continue; // blocked or unparseable URL
+ }
+ const page = await scrapePage(url);
+ if (page) {
+ pages.push(page);
+ }
+ }
+
+ return { pages, blocked: pages.length === 0 };
+}
+
+/**
+ * Discovers a site's representative URLs (homepage + sitemap) and reads them.
+ * Just URL discovery on top of readPages, which does the validated fetching.
+ */
export async function readSite(domain: string): Promise<SiteReadResult> {
let rootUrl: string;
try {
rootUrl = await normalizeAndValidateStartUrl(domain);
} catch {
// Blocked (private/metadata host) or unparseable domain — nothing to read.
- return { rootUrl: `https://${domain}`, pages: [], blocked: true };
+ return { pages: [], blocked: true };
}
const origin = new URL(rootUrl).origin;
// Prefer the sitemap for representative URLs; always include the homepage.
const sitemap = await fetchText(`${origin}/sitemap.xml`);
const discovered = sitemap ? parseSitemapUrls(sitemap, origin) : [];
- const targets = [
- rootUrl,
- ...discovered.filter((url) => url !== rootUrl),
- ].slice(0, MAX_PAGES);
-
- const pages: ScrapedPage[] = [];
- for (const url of targets) {
- const html = await fetchText(url);
- if (!html) {
- continue;
- }
- const text = htmlToText(html).slice(0, PER_PAGE_CHAR_LIMIT);
- if (text.length === 0) {
- continue;
- }
- pages.push({ url, title: extractTitle(html), text });
- }
-
- return { rootUrl, pages, blocked: pages.length === 0 };
+ return readPages([rootUrl, ...discovered.filter((url) => url !== rootUrl)]);
}