Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(content): import a brand page by its URL into a draft (#1288)
Cihanba10b86fcbf8
diff --git a/server/src/lib/page-content.js b/server/src/lib/page-content.jsnew file mode 100644index 000000000..038d6b988--- /dev/null+++ b/server/src/lib/page-content.js@@ -0,0 +1,78 @@+/**+ * A page's own content, for the Content Editor to optimize (CE-003: Optimize+ * content › Page URL): its main text without the site's frame, its title,+ * and its schema JSON-LD. The editor cleans it again, as it does a paste;+ * this only picks what belongs to the page.+ */++import * as cheerio from 'cheerio';++/** The most HTML a page brings into a draft, as the editor's body limit. */+export const MAX_CONTENT_LENGTH = 500_000;++/** The most JSON-LD kept, and the most characters each. */+const MAX_JSON_LD = 5;+const MAX_JSON_LD_LENGTH = 50_000;++/** Where a page's own content usually is, the most telling first. */+const MAIN_SELECTORS = ['main', 'article', '[role="main"]', '#content', '#main'];++/** The site's frame and what can't be content. */+const FRAME =+ 'script, style, noscript, template, iframe, object, embed, form, button, input, select, textarea, svg, canvas, nav, header, footer, aside, [role="navigation"], [role="banner"], [role="contentinfo"], [aria-hidden="true"]';++function clean(text) {+ return (text ?? '').replace(/\s+/g, ' ').trim();+}++/**+ * @param {string} html the page as fetched+ * @param {string} pageUrl the page's address after redirects, for its links+ * @returns {{ title: string, html: string }} html: the content, then its JSON-LD scripts+ */+export function extractPageContent(html, pageUrl) {+ const $ = cheerio.load(html);++ const jsonLd = $('script[type="application/ld+json"]')+ .map((_, el) => $(el).text().trim())+ .get()+ .filter((text) => text && text.length <= MAX_JSON_LD_LENGTH)+ .slice(0, MAX_JSON_LD);++ const h1 = clean($('h1').first().text());+ const title =+ h1 || clean($('meta[property="og:title"]').attr('content')) || clean($('title').first().text());++ const main = MAIN_SELECTORS.map((selector) => $(selector).first()).find((el) => el.length);+ const root = main ?? $('body');+ root.find(FRAME).remove();++ // Links and images keep working outside the page: relative addresses+ // become absolute; anything else the editor's cleaning decides on.+ root.find('a[href]').each((_, el) => {+ const href = $(el).attr('href');+ try {+ $(el).attr('href', new URL(href, pageUrl).toString());+ } catch {+ $(el).removeAttr('href');+ }+ });+ root.find('img[src]').each((_, el) => {+ try {+ $(el).attr('src', new URL($(el).attr('src'), pageUrl).toString());+ } catch {+ $(el).remove();+ }+ });++ let content = (root.html() ?? '').trim();+ // A title above the content (a hero outside <main>) heads it, as the+ // draft's title is its first heading.+ if (title && root.find('h1').length === 0) {+ content = `<h1>${$('<i>').text(title).html()}</h1>${content}`;+ }+ const schema = jsonLd+ .map((text) => `<script type="application/ld+json">${text.replace(/</g, '\\u003c')}</script>`)+ .join('');+ return { title, html: `${content}${schema}` };+}diff --git a/server/src/lib/page-content.test.js b/server/src/lib/page-content.test.jsnew file mode 100644index 000000000..09d7382e9--- /dev/null+++ b/server/src/lib/page-content.test.js@@ -0,0 +1,68 @@+import { describe, expect, it } from 'vitest';+import { extractPageContent } from './page-content.js';++/** A page brought into the Content Editor: its own content, not the site's frame. */++const PAGE = `<!doctype html><html><head>+ <title>Pricing | Northwind</title>+ <meta property="og:title" content="Northwind pricing">+ <script type="application/ld+json">{"@type":"Product","name":"Northwind <CRM>"}</script>+ <script>track()</script>+</head><body>+ <header><nav><a href="/">Home</a></nav></header>+ <main>+ <h1>Plans for every team</h1>+ <p>Start <a href="/signup?plan=pro">free</a>, then <strong>scale</strong>.</p>+ <img src="/img/plans.png" alt="Plans">+ <form><input name="email"><button>Go</button></form>+ <aside>Related posts</aside>+ </main>+ <footer>© Northwind</footer>+</body></html>`;++describe('extractPageContent', () => {+ it('keeps the main content, without the site’s frame, forms or scripts', () => {+ const { title, html } = extractPageContent(PAGE, 'https://northwind.example/pricing');+ expect(title).toBe('Plans for every team');+ expect(html).toContain('<h1>Plans for every team</h1>');+ expect(html).toContain('<strong>scale</strong>');+ for (const gone of ['Home', 'Related posts', '© Northwind', '<form', 'track()']) {+ expect(html).not.toContain(gone);+ }+ });++ it('makes links and images absolute, against the page’s address', () => {+ const { html } = extractPageContent(PAGE, 'https://northwind.example/pricing');+ expect(html).toContain('href="https://northwind.example/signup?plan=pro"');+ expect(html).toContain('src="https://northwind.example/img/plans.png"');+ });++ it('keeps the schema JSON-LD after the content, safe inside its script', () => {+ const { html } = extractPageContent(PAGE, 'https://northwind.example/pricing');+ expect(+ html.endsWith(+ '<script type="application/ld+json">{"@type":"Product","name":"Northwind \\u003cCRM>"}</script>',+ ),+ ).toBe(true);+ });++ it('reads an article, or the body, when there is no <main>', () => {+ const article = extractPageContent(+ '<body><nav>Menu</nav><article><h2>Guide</h2><p>Steps</p></article></body>',+ 'https://acme.example/guide',+ );+ // No H1 and no title anywhere: nothing to head it with.+ expect(article.html).toBe('<h2>Guide</h2><p>Steps</p>');+ const body = extractPageContent('<body><p>Just text</p></body>', 'https://acme.example/x');+ expect(body.html).toBe('<p>Just text</p>');+ });++ it('heads the content with the page title when its H1 sits outside it', () => {+ const { html, title } = extractPageContent(+ '<head><title>Lisbon & more</title></head><body><div class="hero"><h1>Lisbon & Porto</h1></div><main><p>Days out.</p></main></body>',+ 'https://travel.example/lisbon',+ );+ expect(title).toBe('Lisbon & Porto');+ expect(html).toBe('<h1>Lisbon & Porto</h1><p>Days out.</p>');+ });+});diff --git a/server/src/lib/safe-fetch.js b/server/src/lib/safe-fetch.jsnew file mode 100644index 000000000..50e5970c3--- /dev/null+++ b/server/src/lib/safe-fetch.js@@ -0,0 +1,255 @@+/**+ * Fetching a web page a person names (Content Editor CE-003: optimize a page+ * by its URL) without letting the address reach anything private.+ *+ * The address is a user's, so it is checked like one:+ * * http(s) only, on the standard ports, with no credentials in it;+ * * every address its host resolves to must be public. The check runs in+ * the connection's own DNS lookup, so the address connected to is the+ * address checked: a name can't resolve to a public address for a check+ * and a private one for the request (DNS rebinding);+ * * redirects are followed by hand, at most MAX_REDIRECTS, each one checked+ * the same way;+ * * the whole fetch has a time limit, and the body a size limit.+ *+ * Only HTML comes back, decoded as text. Failures carry a code the caller+ * can name to the person (PageFetchError.code).+ */++import dns from 'node:dns';+import http from 'node:http';+import https from 'node:https';+import net from 'node:net';+import { TextDecoder } from 'node:util';+import zlib from 'node:zlib';++export const FETCH_TIMEOUT_MS = 10_000;+export const MAX_PAGE_BYTES = 3 * 1024 * 1024;+export const MAX_REDIRECTS = 5;++const USER_AGENT = 'Mozilla/5.0 (compatible; AnsvisorBot/1.0; +https://ansvisor.com)';++export class PageFetchError extends Error {+ /**+ * @param {'invalid_url'|'blocked'|'unreachable'|'timeout'|'too_large'|'not_html'|'http_error'|'too_many_redirects'} code+ * @param {string} message+ * @param {{ status?: number }} [extra]+ */+ constructor(code, message, extra = {}) {+ super(message);+ this.code = code;+ this.status = extra.status;+ }+}++// Private, local, shared, documentation and reserved ranges (RFC 6890 and+// friends), and the IPv6 forms that carry an IPv4 address inside. An+// IPv4-mapped address (::ffff:a.b.c.d) is checked against the IPv4 ranges by+// BlockList itself; a rule for ::ffff:0:0/96 would match every IPv4 address.+const BLOCKED = new net.BlockList();+for (const [address, prefix] of [+ ['0.0.0.0', 8],+ ['10.0.0.0', 8],+ ['100.64.0.0', 10],+ ['127.0.0.0', 8],+ ['169.254.0.0', 16],+ ['172.16.0.0', 12],+ ['192.0.0.0', 24],+ ['192.0.2.0', 24],+ ['192.88.99.0', 24],+ ['192.168.0.0', 16],+ ['198.18.0.0', 15],+ ['198.51.100.0', 24],+ ['203.0.113.0', 24],+ ['224.0.0.0', 4],+ ['240.0.0.0', 4],+]) {+ BLOCKED.addSubnet(address, prefix, 'ipv4');+}+for (const [address, prefix] of [+ ['::', 128],+ ['::1', 128],+ ['64:ff9b::', 96],+ ['100::', 64],+ ['2001::', 32],+ ['2001:db8::', 32],+ ['2002::', 16],+ ['fc00::', 7],+ ['fe80::', 10],+ ['ff00::', 8],+]) {+ BLOCKED.addSubnet(address, prefix, 'ipv6');+}++/** Whether an IP address is one a public web page may live at. */+export function isPublicAddress(address) {+ const family = net.isIP(address);+ if (family === 0) return false;+ return !BLOCKED.check(address, family === 6 ? 'ipv6' : 'ipv4');+}++/**+ * A DNS lookup that refuses names resolving to any address that isn't+ * public, in the signature http.request's `lookup` option takes.+ */+export function publicOnlyLookup(hostname, options, callback) {+ const opts = typeof options === 'function' ? {} : options;+ const done = typeof options === 'function' ? options : callback;+ dns.lookup(hostname, { ...opts, all: true, verbatim: true }, (err, addresses) => {+ if (err) return done(err);+ if (!addresses.length || addresses.some((a) => !isPublicAddress(a.address))) {+ return done(+ new PageFetchError('blocked', `${hostname} does not resolve to a public address`),+ );+ }+ if (opts.all) return done(null, addresses);+ return done(null, addresses[0].address, addresses[0].family);+ });+}++/** The standard web ports ('' is the scheme's default). */+const WEB_PORTS = ['', '80', '443'];++/** A URL a person gave, checked before anything is fetched; throws 'invalid_url'. */+export function checkPageUrl(value, ports = WEB_PORTS) {+ let url;+ try {+ url = new URL(String(value).trim());+ } catch {+ throw new PageFetchError('invalid_url', 'Not a URL');+ }+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {+ throw new PageFetchError('invalid_url', 'Only http and https pages can be read');+ }+ if (url.username || url.password) {+ throw new PageFetchError('invalid_url', 'URLs with credentials are not read');+ }+ if (!ports.includes(url.port)) {+ throw new PageFetchError('invalid_url', 'Only the standard web ports are read');+ }+ // An address written as an IP connects without a DNS lookup, so the+ // lookup's check never sees it: it is checked here. The URL parser has+ // already turned 2130706433 or 0x7f.1 into 127.0.0.1.+ const host = url.hostname.replace(/^\[|\]$/g, '');+ if (net.isIP(host) && !isPublicAddress(host)) {+ throw new PageFetchError('blocked', `${host} is not a public address`);+ }+ url.hash = '';+ return url;+}++function charsetOf(contentType) {+ const match = /charset=["']?([\w-]+)/i.exec(contentType ?? '');+ return match ? match[1].toLowerCase() : 'utf-8';+}++function decode(buffer, contentType) {+ try {+ return new TextDecoder(charsetOf(contentType)).decode(buffer);+ } catch {+ return new TextDecoder('utf-8').decode(buffer);+ }+}++/** One request, no redirects followed: the response, or where it redirects to. */+function requestOnce(url, { signal, lookup, maxBytes }) {+ return new Promise((resolve, reject) => {+ const client = url.protocol === 'https:' ? https : http;+ const req = client.request(+ url,+ {+ method: 'GET',+ agent: false,+ lookup,+ signal,+ headers: {+ 'user-agent': USER_AGENT,+ accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.1',+ 'accept-encoding': 'gzip, deflate, br',+ },+ },+ (res) => {+ const status = res.statusCode ?? 0;+ if (status >= 300 && status < 400 && res.headers.location) {+ res.resume();+ resolve({ redirect: res.headers.location, status });+ return;+ }+ if (status < 200 || status >= 300) {+ res.resume();+ reject(new PageFetchError('http_error', `The page answered ${status}`, { status }));+ return;+ }+ const contentType = res.headers['content-type'] ?? '';+ if (!/^\s*(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {+ res.resume();+ reject(new PageFetchError('not_html', `The address is ${contentType || 'not a page'}`));+ return;+ }+ const encoding = String(res.headers['content-encoding'] ?? '').toLowerCase();+ let body = res;+ if (encoding === 'gzip' || encoding === 'x-gzip') body = res.pipe(zlib.createGunzip());+ else if (encoding === 'deflate') body = res.pipe(zlib.createInflate());+ else if (encoding === 'br') body = res.pipe(zlib.createBrotliDecompress());++ const chunks = [];+ let size = 0;+ body.on('data', (chunk) => {+ size += chunk.length;+ if (size > maxBytes) {+ req.destroy();+ reject(new PageFetchError('too_large', 'The page is too large to read'));+ return;+ }+ chunks.push(chunk);+ });+ body.on('end', () =>+ resolve({ status, contentType, body: decode(Buffer.concat(chunks), contentType) }),+ );+ body.on('error', (err) => reject(err));+ },+ );+ req.on('error', reject);+ req.end();+ });+}++/**+ * A web page's HTML, fetched safely (see above).+ *+ * @param {string} address+ * @param {{ timeoutMs?: number, maxBytes?: number, maxRedirects?: number, lookup?: Function, ports?: string[] }} [opts]+ * `lookup` and `ports` replace the public-only lookup and the web ports;+ * tests use them to reach a local server.+ * @returns {Promise<{ url: string, status: number, html: string }>} url is the address after redirects+ */+export async function fetchPage(address, opts = {}) {+ const {+ timeoutMs = FETCH_TIMEOUT_MS,+ maxBytes = MAX_PAGE_BYTES,+ maxRedirects = MAX_REDIRECTS,+ lookup = publicOnlyLookup,+ ports = WEB_PORTS,+ } = opts;+ const signal = AbortSignal.timeout(timeoutMs);+ let url = checkPageUrl(address, ports);+ for (let hops = 0; ; hops += 1) {+ let res;+ try {+ res = await requestOnce(url, { signal, lookup, maxBytes });+ } catch (err) {+ if (err instanceof PageFetchError) throw err;+ if (signal.aborted) throw new PageFetchError('timeout', 'The page took too long to answer');+ if (err?.cause instanceof PageFetchError) throw err.cause;+ throw new PageFetchError(+ 'unreachable',+ `The page could not be reached (${err?.code ?? err?.message})`,+ );+ }+ if (!res.redirect) return { url: url.toString(), status: res.status, html: res.body };+ if (hops >= maxRedirects) {+ throw new PageFetchError('too_many_redirects', 'The page redirects too many times');+ }+ url = checkPageUrl(new URL(res.redirect, url).toString(), ports);+ }+}diff --git a/server/src/lib/safe-fetch.test.js b/server/src/lib/safe-fetch.test.jsnew file mode 100644index 000000000..9b8e3824d--- /dev/null+++ b/server/src/lib/safe-fetch.test.js@@ -0,0 +1,196 @@+import http from 'node:http';+import zlib from 'node:zlib';+import { afterAll, beforeAll, describe, expect, it } from 'vitest';+import {+ PageFetchError,+ checkPageUrl,+ fetchPage,+ isPublicAddress,+ publicOnlyLookup,+} from './safe-fetch.js';++/**+ * Fetching a page a person names (Content Editor CE-003) must never reach a+ * private address, and stops at time, size and redirect limits.+ */++describe('isPublicAddress', () => {+ it('takes public addresses', () => {+ for (const address of ['8.8.8.8', '93.184.216.34', '2606:4700:4700::1111', '::ffff:8.8.8.8']) {+ expect(isPublicAddress(address)).toBe(true);+ }+ });++ it('refuses private, local, metadata, shared, reserved and mapped addresses', () => {+ for (const address of [+ '127.0.0.1',+ '10.1.2.3',+ '172.16.5.4',+ '192.168.1.1',+ '169.254.169.254',+ '100.64.0.1',+ '0.0.0.0',+ '224.0.0.1',+ '255.255.255.255',+ '::1',+ '::',+ 'fe80::1',+ 'fd12:3456::1',+ '::ffff:127.0.0.1',+ '::ffff:10.0.0.1',+ '64:ff9b::7f00:1',+ 'not-an-ip',+ ]) {+ expect(isPublicAddress(address), address).toBe(false);+ }+ });+});++describe('checkPageUrl', () => {+ it('takes http(s) on the web ports, without the fragment', () => {+ expect(checkPageUrl(' https://acme.example/pricing#plans ').toString()).toBe(+ 'https://acme.example/pricing',+ );+ expect(checkPageUrl('http://acme.example:80/').protocol).toBe('http:');+ });++ it.each([+ ['ftp://acme.example/x'],+ ['file:///etc/passwd'],+ ['javascript:alert(1)'],+ ['https://user:[email protected]/'],+ ['https://acme.example:8443/'],+ ['not a url'],+ ])('refuses %s', (value) => {+ expect(() => checkPageUrl(value)).toThrow(PageFetchError);+ });+});++describe('fetchPage on the public-only lookup', () => {+ it('refuses a host that resolves to a private address, before any request', async () => {+ // Written as IPs, in any form the URL parser reads, they never reach a lookup.+ for (const address of [+ 'http://localhost/',+ 'http://127.0.0.1/',+ 'http://[::1]/',+ 'http://2130706433/',+ 'http://0x7f.1/',+ 'http://169.254.169.254/latest/meta-data/',+ 'http://[::ffff:127.0.0.1]/',+ ]) {+ await expect(fetchPage(address), address).rejects.toMatchObject({ code: 'blocked' });+ }+ });++ it('checks every address a name resolves to', async () => {+ const result = await new Promise((resolve) =>+ publicOnlyLookup('localhost', { all: true }, (err, addresses) => resolve({ err, addresses })),+ );+ expect(result.err).toBeInstanceOf(PageFetchError);+ expect(result.addresses).toBeUndefined();+ });+});++describe('fetchPage against a local site', () => {+ let server;+ let port;+ const routes = {+ '/page': (res) => {+ res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });+ res.end('<h1>Pricing</h1><p>Plans ünd prices</p>');+ },+ '/hop': (res) => {+ res.writeHead(301, { location: '/page' });+ res.end();+ },+ '/loop': (res) => {+ res.writeHead(302, { location: '/loop' });+ res.end();+ },+ '/to-ftp': (res) => {+ res.writeHead(302, { location: 'ftp://acme.example/' });+ res.end();+ },+ '/to-private': (res) => {+ res.writeHead(302, { location: 'http://localhost/admin' });+ res.end();+ },+ '/to-ip': (res) => {+ res.writeHead(302, { location: 'http://127.0.0.1/admin' });+ res.end();+ },+ '/gzip': (res) => {+ res.writeHead(200, { 'content-type': 'text/html', 'content-encoding': 'gzip' });+ res.end(zlib.gzipSync('<p>squeezed</p>'));+ },+ '/pdf': (res) => {+ res.writeHead(200, { 'content-type': 'application/pdf' });+ res.end('%PDF');+ },+ '/missing': (res) => {+ res.writeHead(404, { 'content-type': 'text/html' });+ res.end('nope');+ },+ '/big': (res) => {+ res.writeHead(200, { 'content-type': 'text/html' });+ res.end('x'.repeat(5_000));+ },+ '/slow': (res) => {+ setTimeout(() => {+ res.writeHead(200, { 'content-type': 'text/html' });+ res.end('late');+ }, 300);+ },+ };++ beforeAll(async () => {+ server = http.createServer((req, res) => (routes[req.url] ?? routes['/missing'])(res));+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));+ port = String(server.address().port);+ });+ afterAll(() => new Promise((resolve) => server.close(resolve)));++ // site.test is the local server; any other name goes through the real check.+ const lookup = (hostname, options, callback) => {+ const done = typeof options === 'function' ? options : callback;+ const all = typeof options === 'object' && options.all;+ if (hostname !== 'site.test') return publicOnlyLookup(hostname, options, callback);+ return all ? done(null, [{ address: '127.0.0.1', family: 4 }]) : done(null, '127.0.0.1', 4);+ };+ const get = (path, opts = {}) =>+ fetchPage(`http://site.test:${port}${path}`, {+ lookup,+ ports: ['', '80', '443', port],+ ...opts,+ });++ it('reads a page as text, in its charset', async () => {+ await expect(get('/page')).resolves.toEqual({+ url: `http://site.test:${port}/page`,+ status: 200,+ html: '<h1>Pricing</h1><p>Plans ünd prices</p>',+ });+ });++ it('follows redirects to where they end, and says where that is', async () => {+ await expect(get('/hop')).resolves.toMatchObject({ url: `http://site.test:${port}/page` });+ });++ it('checks each redirect like the first address', async () => {+ await expect(get('/to-ftp')).rejects.toMatchObject({ code: 'invalid_url' });+ await expect(get('/to-private')).rejects.toMatchObject({ code: 'blocked' });+ await expect(get('/to-ip')).rejects.toMatchObject({ code: 'blocked' });+ await expect(get('/loop')).rejects.toMatchObject({ code: 'too_many_redirects' });+ });++ it('opens compressed pages', async () => {+ await expect(get('/gzip')).resolves.toMatchObject({ html: '<p>squeezed</p>' });+ });++ it('says why it brought nothing back', async () => {+ await expect(get('/pdf')).rejects.toMatchObject({ code: 'not_html' });+ await expect(get('/missing')).rejects.toMatchObject({ code: 'http_error', status: 404 });+ await expect(get('/big', { maxBytes: 1_000 })).rejects.toMatchObject({ code: 'too_large' });+ await expect(get('/slow', { timeoutMs: 50 })).rejects.toMatchObject({ code: 'timeout' });+ });+});diff --git a/server/src/routes/content-pages.js b/server/src/routes/content-pages.jsnew file mode 100644index 000000000..de5267e56--- /dev/null+++ b/server/src/routes/content-pages.js@@ -0,0 +1,85 @@+import { Router } from 'express';+import { z } from 'zod';+import supabaseAdmin from '../config/supabase.js';+import { assertBrandAccess } from '../lib/access.js';+import { onBrandDomain } from '../lib/audit/host.js';+import { MAX_CONTENT_LENGTH, extractPageContent } from '../lib/page-content.js';+import { PageFetchError, fetchPage } from '../lib/safe-fetch.js';++/**+ * Pages brought into the Content Editor (CE-003): Optimize content › Page+ * URL, and Start's "Optimize this page instead". Only the brand's own pages+ * (its domains or their subdomains) are read: the editor optimizes the+ * brand's content, not anyone else's, and it keeps the fetch to sites the+ * brand named. The fetch itself refuses private addresses (safe-fetch.js).+ */++const router = Router();++const importBody = z.object({+ // guid, not uuid: brand ids aren't all RFC 4122 versions (seeded ones).+ brandId: z.guid(),+ url: z.string().trim().min(1).max(2048),+});++/** The person-facing reason a page couldn't be read, by fetch error code. */+const FETCH_STATUS = {+ invalid_url: 400,+ blocked: 400,+ too_many_redirects: 422,+ not_html: 422,+ too_large: 422,+ http_error: 422,+ timeout: 504,+ unreachable: 502,+};++router.post('/import', async (req, res) => {+ const parsed = importBody.safeParse(req.body);+ if (!parsed.success) {+ const urlIssue = parsed.error.issues.some((issue) => issue.path[0] === 'url');+ return res.status(400).json({ error: 'invalid', code: urlIssue ? 'invalid_url' : 'invalid' });+ }+ const { brandId } = parsed.data;+ // A bare host ("acme.com/pricing") is read as https.+ const url = /^[a-z][a-z0-9+.-]*:/i.test(parsed.data.url)+ ? parsed.data.url+ : `https://${parsed.data.url}`;++ try {+ await assertBrandAccess(brandId, req.user.id);+ const { data: domains, error } = await supabaseAdmin+ .from('brand_domains')+ .select('domain')+ .eq('brand_id', brandId);+ if (error) throw error;+ const brandDomains = (domains ?? []).map((d) => d.domain);+ if (!onBrandDomain(url, brandDomains)) {+ return res.status(400).json({ error: 'Not on the brand’s site', code: 'off_site' });+ }++ const page = await fetchPage(url);+ // A redirect off the brand's site isn't the brand's page.+ if (!onBrandDomain(page.url, brandDomains)) {+ return res.status(422).json({ error: 'The page redirects off the site', code: 'off_site' });+ }+ const content = extractPageContent(page.html, page.url);+ if (content.html.length > MAX_CONTENT_LENGTH) {+ return res.status(422).json({ error: 'The page is too large', code: 'too_large' });+ }+ return res.json({ url: page.url, title: content.title, html: content.html });+ } catch (err) {+ if (err instanceof PageFetchError) {+ return res+ .status(FETCH_STATUS[err.code] ?? 502)+ .json({ error: err.message, code: err.code, status: err.status ?? null });+ }+ if (err?.status === 403 || err?.status === 404) {+ return res.status(err.status).json({ error: err.message, code: 'forbidden' });+ }+ req.log.error({ err }, 'content page import error');+ return res.status(500).json({ error: 'Failed to read the page', code: 'failed' });+ }+});++export default router;diff --git a/server/src/routes/content-pages.test.js b/server/src/routes/content-pages.test.jsnew file mode 100644index 000000000..65320b8b3--- /dev/null+++ b/server/src/routes/content-pages.test.js@@ -0,0 +1,137 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import express from 'express';++/**+ * Optimize content › Page URL (Content Editor CE-003): only the brand's own+ * pages are read, and a page that can't be read says why.+ */++// Seeded brands' ids aren't RFC 4122 versions; the route must still take them.+const BRAND = '33333333-3333-3333-3333-333333333333';+let org;+let fetched;+let fetchResult;++vi.mock('../config/supabase.js', () => ({+ default: {+ from: (name) => {+ const result =+ name === 'brands'+ ? { data: { id: BRAND, organization_id: 'o1' }, error: null }+ : {+ data: [{ domain: 'https://www.Northwind.example/' }, { domain: 'northwind.shop' }],+ error: null,+ };+ const chain = {+ select: () => chain,+ eq: () => chain,+ single: async () => result,+ then: (resolve) => resolve(result),+ };+ return chain;+ },+ },+}));+vi.mock('../lib/plan-guard.js', () => ({ getOrgIdForUser: async () => org }));+vi.mock('../lib/safe-fetch.js', async (importOriginal) => {+ const actual = await importOriginal();+ return {+ ...actual,+ fetchPage: async (url) => {+ fetched.push(url);+ if (fetchResult instanceof Error) throw fetchResult;+ return fetchResult;+ },+ };+});++const { default: router } = await import('./content-pages.js');+const { PageFetchError } = await import('../lib/safe-fetch.js');++let server;+let baseUrl;++beforeEach(async () => {+ org = 'o1';+ fetched = [];+ fetchResult = {+ url: 'https://northwind.example/pricing',+ status: 200,+ html: '<main><h1>Pricing</h1><p>Plans</p></main>',+ };+ const app = express();+ app.use(express.json());+ app.use((req, _res, next) => {+ req.user = { id: 'u1' };+ req.log = { error: vi.fn() };+ next();+ });+ app.use('/api/content-pages', router);+ await new Promise((resolve) => {+ server = app.listen(0, resolve);+ });+ baseUrl = `http://127.0.0.1:${server.address().port}/api/content-pages`;+});++afterEach(() => new Promise((resolve) => server.close(resolve)));++const importPage = (url, brandId = BRAND) =>+ fetch(`${baseUrl}/import`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ brandId, url }),+ });++describe('POST /api/content-pages/import', () => {+ it('reads a page of the brand’s, a bare host as https, and returns its content', async () => {+ const res = await importPage('blog.northwind.example/pricing');+ expect(res.status).toBe(200);+ expect(fetched).toEqual(['https://blog.northwind.example/pricing']);+ await expect(res.json()).resolves.toEqual({+ url: 'https://northwind.example/pricing',+ title: 'Pricing',+ html: '<h1>Pricing</h1><p>Plans</p>',+ });+ });++ it('reads no page off the brand’s site', async () => {+ for (const url of [+ 'https://competitor.example/pricing',+ 'https://northwind.example.evil.com/',+ ]) {+ const res = await importPage(url);+ expect(res.status).toBe(400);+ await expect(res.json()).resolves.toMatchObject({ code: 'off_site' });+ }+ expect(fetched).toEqual([]);+ });++ it('refuses a page that redirects off the site', async () => {+ fetchResult = { url: 'https://elsewhere.example/', status: 200, html: '<p>x</p>' };+ const res = await importPage('https://northwind.shop/old');+ expect(res.status).toBe(422);+ await expect(res.json()).resolves.toMatchObject({ code: 'off_site' });+ });++ it('says why a page couldn’t be read', async () => {+ fetchResult = new PageFetchError('http_error', 'The page answered 404', { status: 404 });+ let res = await importPage('https://northwind.example/gone');+ expect(res.status).toBe(422);+ await expect(res.json()).resolves.toMatchObject({ code: 'http_error', status: 404 });++ fetchResult = new PageFetchError('timeout', 'slow');+ res = await importPage('https://northwind.example/slow');+ expect(res.status).toBe(504);+ await expect(res.json()).resolves.toMatchObject({ code: 'timeout' });+ });++ it('refuses another organization’s brand, and an id that isn’t one', async () => {+ org = 'o2';+ let res = await importPage('https://northwind.example/pricing');+ expect(res.status).toBe(403);+ res = await importPage('https://northwind.example/pricing', 'nope');+ expect(res.status).toBe(400);+ await expect(res.json()).resolves.toMatchObject({ code: 'invalid' });+ expect(fetched).toEqual([]);+ });+});diff --git a/server/src/routes/index.js b/server/src/routes/index.jsindex a44ce0cc7..835f1d22b 100644--- a/server/src/routes/index.js+++ b/server/src/routes/index.js@@ -3,6 +3,7 @@ import promptRoutes from './prompts.js';import trackingRoutes from './tracking.js';import volumeRoutes from './volumes.js';import contentRoutes from './content.js';+import contentPageRoutes from './content-pages.js';import competitorRoutes from './competitors.js';import topicRoutes from './topics.js';import auditRoutes from './audits.js';@@ -19,6 +20,7 @@ router.use('/prompts', promptRoutes);router.use('/tracking', trackingRoutes);router.use('/volumes', volumeRoutes);router.use('/content', contentRoutes);+router.use('/content-pages', contentPageRoutes);router.use('/competitors', competitorRoutes);router.use('/topics', topicRoutes);router.use('/audits', auditRoutes);diff --git a/web/messages/en.json b/web/messages/en.jsonindex eb04408a1..34b67d28f 100644--- a/web/messages/en.json+++ b/web/messages/en.json@@ -3116,7 +3116,7 @@},"optimize": {"title": "Optimize content",- "hint": "Pasted text or a file"+ "hint": "A page URL, pasted text or a file"}},"startFrom": "Start from",@@ -3146,6 +3146,24 @@"retry": "Try again","content": {"label": "Content",+ "url": "Page URL",+ "urlPlaceholder": "https://yourbrand.com/page",+ "urlHint": "A page on your brand's site. Its main text, headings, links and schema come in; menus and footers stay out.",+ "import": "Import",+ "importing": "Reading…",+ "urlErrors": {+ "off_site": "Use a page on your brand's site. Its domains are in the brand settings.",+ "invalid_url": "Enter a page's address, like https://yourbrand.com/pricing.",+ "blocked": "That address can't be read.",+ "not_html": "That address isn't a web page.",+ "http_error": "The page couldn't be opened.",+ "http_status": "The page answered {status}.",+ "too_large": "The page is too large to bring in.",+ "timeout": "The page took too long to answer.",+ "unreachable": "The page couldn't be reached.",+ "too_many_redirects": "The page redirects too many times.",+ "failed": "Couldn't read the page. Try again."+ },"paste": "Paste text","file": "Upload file","pastePlaceholder": "Paste from Word, Google Docs or a web page. Headings, lists, links and tables are kept.",@@ -3170,8 +3188,9 @@"pageType": "Page type","detectedTitle": "Picked from the title.","detectedContent": "Picked from the content.",- "detectedPage": "Picked from the page's address.",+ "detectedPage": "Picked from the page.","openEditor": "Open editor",+ "pageUnread": "Couldn't read {page}: the draft starts with its title.","failed": "Couldn't start the draft. Try again."},"overlap": {diff --git a/web/src/components/content/new-content-dialog.test.tsx b/web/src/components/content/new-content-dialog.test.tsxindex 6b5dd2a43..a0c64568c 100644--- a/web/src/components/content/new-content-dialog.test.tsx+++ b/web/src/components/content/new-content-dialog.test.tsx@@ -9,14 +9,18 @@ import { NewContentDialog } from './new-content-dialog';const getContentStartOptions = vi.fn();const createContentDraft = vi.fn();const getContentOverlap = vi.fn();+const importContentPage = vi.fn();vi.mock('@/lib/actions/content-drafts', () => ({+ importContentPage: (...args: unknown[]) => importContentPage(...args),getContentStartOptions: (...args: unknown[]) => getContentStartOptions(...args),createContentDraft: (...args: unknown[]) => createContentDraft(...args),getContentOverlap: (...args: unknown[]) => getContentOverlap(...args),}));const push = vi.fn();vi.mock('@/i18n/navigation', () => ({ useRouter: () => ({ push }) }));-const toast = vi.hoisted(() => Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn() }));+const toast = vi.hoisted(() =>+ Object.assign(vi.fn(), { success: vi.fn(), error: vi.fn(), warning: vi.fn() }),+);vi.mock('sonner', () => ({ toast }));(@@ -80,6 +84,8 @@ const click = async (el: Element | undefined) => {(el as HTMLElement).click();});};+const tab = (name: string) =>+ [...document.querySelectorAll('[role="tab"]')].find((t) => t.textContent === name);const checkedType = () =>document.querySelector('[aria-labelledby="start-page-type"] [aria-checked="true"]')?.textContent;@@ -90,6 +96,8 @@ beforeEach(() => {getContentStartOptions.mockReset().mockResolvedValue(OPTIONS);createContentDraft.mockReset().mockResolvedValue('d9');getContentOverlap.mockReset().mockResolvedValue({ totalPrompts: 2, pages: [] });+ importContentPage.mockReset().mockResolvedValue({ error: 'failed' });+ toast.warning.mockReset();push.mockReset();toast.error.mockReset();});@@ -147,6 +155,7 @@ describe('New content', () => {it('optimizes pasted content, cleaned, and reads its page type from it', async () => {await render('ecommerce');await click(radio('Optimize content'));+ await click(tab('Paste text'));const box = document.querySelector('textarea')!;const paste = new Event('paste', { bubbles: true, cancelable: true });Object.defineProperty(paste, 'clipboardData', {@@ -182,6 +191,7 @@ describe('New content', () => {it('takes typed plain text as paragraphs, and a blog post when nothing tells the type', async () => {await render();await click(radio('Optimize content'));+ await click(tab('Paste text'));const box = document.querySelector('textarea')!;const setValue = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value')!.set!;await act(async () => {@@ -201,9 +211,7 @@ describe('New content', () => {it('reads an uploaded Markdown file, and refuses a spreadsheet', async () => {await render();await click(radio('Optimize content'));- await click(- [...document.querySelectorAll('[role="tab"]')].find((t) => t.textContent === 'Upload file'),- );+ await click(tab('Upload file'));const input = document.querySelector<HTMLInputElement>('input[type="file"]')!;const upload = async (file: File) => {Object.defineProperty(input, 'files', { value: [file], configurable: true });@@ -231,6 +239,66 @@ describe('New content', () => {expect(button('Pick an opportunity')).toBeDefined();});+ describe('Optimize content › Page URL', () => {+ async function importUrl(url: string) {+ await render();+ await click(radio('Optimize content'));+ const input = document.querySelector<HTMLInputElement>('input[aria-label="Page URL"]')!;+ const setValue = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!;+ await act(async () => {+ setValue.call(input, url);+ input.dispatchEvent(new Event('input', { bubbles: true }));+ });+ await click(button('Import'));+ await act(async () => {});+ }++ it('reads a page of the brand’s, cleaned, and starts from its address', async () => {+ importContentPage.mockResolvedValue({+ page: {+ url: 'https://northwind.example/pricing',+ title: 'Pricing',+ html: '<h1>Pricing</h1><p onclick="x()">Plans</p><script>track()</script>',+ },+ });+ await importUrl('northwind.example/pricing');+ expect(importContentPage).toHaveBeenCalledWith('b1', 'northwind.example/pricing');+ expect(document.body.textContent).toContain('/pricing · 1 heading');++ await click(button('Continue'));+ expect(document.body.textContent).toContain('Optimize: Pricing');+ expect(checkedType()).toBe('Pricing');+ expect(document.body.textContent).toContain('Picked from the page.');+ // Content brought to optimize has no prompts to check.+ expect(getContentOverlap).not.toHaveBeenCalled();++ await click(button('Open editor'));+ expect(createContentDraft).toHaveBeenCalledWith('b1', {+ method: 'optimize',+ title: 'Pricing',+ pageType: 'pricing',+ typePicked: false,+ source: { kind: 'content' },+ bodyHtml: '<h1>Pricing</h1><p>Plans</p>',+ sourceUrl: 'https://northwind.example/pricing',+ });+ });++ it('says why a page couldn’t be read', async () => {+ importContentPage.mockResolvedValue({ error: 'off_site' });+ await importUrl('https://competitor.example/pricing');+ expect(document.querySelector('[role="alert"]')!.textContent).toBe(+ "Use a page on your brand's site. Its domains are in the brand settings.",+ );+ expect(button('Continue')!.disabled).toBe(true);++ importContentPage.mockResolvedValue({ error: 'http_error', status: 404 });+ await click(button('Import'));+ await act(async () => {});+ expect(document.querySelector('[role="alert"]')!.textContent).toBe('The page answered 404.');+ });+ });+describe('the overlap check', () => {const GUIDE = {url: 'https://acme.example/guides/espresso-machines-for-beginners/',@@ -278,9 +346,18 @@ describe('New content', () => {// The draft becomes that page's: its title, and the type its address shows.expect(document.body.textContent).toContain('Optimize: Espresso machines for beginners');expect(checkedType()).toBe('Guide');- expect(document.body.textContent).toContain("Picked from the page's address.");+ expect(document.body.textContent).toContain('Picked from the page.');+ importContentPage.mockResolvedValue({+ page: {+ url: GUIDE.url,+ title: 'Espresso machines for beginners',+ html: '<h1>Espresso machines for beginners</h1><p style="color:red">Start here.</p>',+ },+ });await click(button('Open editor'));+ // The page's live text comes in, cleaned like a paste.+ expect(importContentPage).toHaveBeenCalledWith('b1', GUIDE.url);expect(createContentDraft).toHaveBeenCalledWith('b1', {method: 'optimize',title: 'Espresso machines for beginners',@@ -288,6 +365,20 @@ describe('New content', () => {// Read from the page's address, so it follows the page.typePicked: false,source: { kind: 'opportunity', id: 'o1' },+ bodyHtml: '<h1>Espresso machines for beginners</h1><p>Start here.</p>',+ sourceUrl: GUIDE.url,+ });+ });++ it('starts an existing page with its title when its live text can’t be read', async () => {+ getContentOverlap.mockResolvedValue({ totalPrompts: 4, pages: [GUIDE] });+ await startFromOpportunity();+ await click(button('Open editor'));+ expect(toast.warning).toHaveBeenCalledWith(+ "Couldn't read /guides/espresso-machines-for-beginners: the draft starts with its title.",+ );+ expect(createContentDraft.mock.calls[0][1]).toMatchObject({+ method: 'optimize',bodyHtml: undefined,sourceUrl: GUIDE.url,});diff --git a/web/src/components/content/new-content-dialog.tsx b/web/src/components/content/new-content-dialog.tsxindex 5e4ed04e8..87e2b4e21 100644--- a/web/src/components/content/new-content-dialog.tsx+++ b/web/src/components/content/new-content-dialog.tsx@@ -25,6 +25,7 @@ import {DialogHeader,DialogTitle,} from '@/components/ui/dialog';+import { Input } from '@/components/ui/input';import { Label } from '@/components/ui/label';import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs';import { Textarea } from '@/components/ui/textarea';@@ -32,6 +33,7 @@ import {createContentDraft,getContentOverlap,getContentStartOptions,+ importContentPage,} from '@/lib/actions/content-drafts';import { MAX_ATTACHMENT_BYTES, textToHtml } from '@/lib/content/attachments';import { cleanHtml } from '@/lib/content/clean-html';@@ -41,6 +43,7 @@ import {type DraftMethod,type DraftSource,type DraftStartOptions,+ type PageImportError,type PageType,} from '@/lib/content/drafts';import { DOCUMENT_ACCEPT, readDocumentFile } from '@/lib/content/import-document';@@ -76,9 +79,14 @@ interface BroughtContent extends ContentSummary {html: string;/** The file's name; none for a paste. */fileName?: string;+ /** The page's address, for content read from a URL. */+ sourceUrl?: string;imagesRemoved: number;}+/** Where content to optimize comes from. */+type ContentFrom = 'url' | 'paste' | 'file';+/** Where Start's first page type came from, if it was read from somewhere. */type DetectedFrom = 'title' | 'content' | 'page' | null;@@ -88,6 +96,8 @@ interface StartChoice {title: string;source: DraftSource;bodyHtml?: string;+ /** The page the content was read from, for content brought by URL. */+ sourceUrl?: string;pageType: PageType;detectedFrom: DetectedFrom;/** The person picked the type from the chips; otherwise it follows the page. */@@ -133,7 +143,9 @@ export function NewContentDialog({const [method, setMethod] = useState<DraftMethod>('create');const [kind, setKind] = useState<SourceKind>('opportunity');const [picked, setPicked] = useState<PickerItem | null>(null);- const [from, setFrom] = useState<'paste' | 'file'>('paste');+ const [from, setFrom] = useState<ContentFrom>('url');+ const [pageUrl, setPageUrl] = useState('');+ const [urlError, setUrlError] = useState<{ code: PageImportError; status?: number } | null>(null);const [pastedText, setPastedText] = useState('');const [content, setContent] = useState<BroughtContent | null>(null);const [reading, setReading] = useState(false);@@ -192,7 +204,9 @@ export function NewContentDialog({setMethod('create');setKind('opportunity');setPicked(null);- setFrom('paste');+ setFrom('url');+ setPageUrl('');+ setUrlError(null);setPastedText('');setContent(null);setStart(null);@@ -220,13 +234,40 @@ export function NewContentDialog({meta: t('topicPrompts', { count: topic.promptCount }),}));- const bring = (html: string, imagesRemoved: number, fileName?: string) => {- const summary = summarizeContent(html, fileName ? fileTitle(fileName) : '');+ const bring = (+ html: string,+ imagesRemoved: number,+ origin: { fileName?: string; sourceUrl?: string; title?: string } = {},+ ) => {+ const fallback = origin.title || (origin.fileName ? fileTitle(origin.fileName) : '');+ const summary = summarizeContent(html, fallback);if (!summary.characters) {toast.error(t('content.empty'));return;}- setContent({ ...summary, html, fileName, imagesRemoved });+ setContent({ ...summary, html, imagesRemoved, ...origin });+ };++ /** A page of the brand's, read by its URL and cleaned like a paste. */+ const readPage = async () => {+ const address = pageUrl.trim();+ if (!address) return;+ setReading(true);+ setUrlError(null);+ try {+ const read = await importContentPage(brandId, address);+ if ('error' in read) {+ setUrlError({ code: read.error, status: read.status });+ return;+ }+ const cleaned = cleanHtml(read.page.html, { keepImages: true });+ bring(cleaned.html, cleaned.imagesRemoved, {+ sourceUrl: read.page.url,+ title: read.page.title,+ });+ } finally {+ setReading(false);+ }};const onPaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {@@ -249,7 +290,7 @@ export function NewContentDialog({try {const doc = await readDocumentFile(file);if (!doc) toast.error(t('content.unsupported', { name: file.name }));- else bring(doc.html, doc.imagesRemoved, file.name);+ else bring(doc.html, doc.imagesRemoved, { fileName: file.name });} catch (err) {console.error('Failed to read a file for New content:', err);toast.error(t('content.readFailed', { name: file.name }));@@ -286,10 +327,14 @@ export function NewContentDialog({content ??(typed ? { ...summarizeContent(textToHtml(typed)), html: textToHtml(typed) } : null);if (!brought) return;- const type = startPageType({ title: brought.title, html: brought.html }, siteType);+ const sourceUrl = 'sourceUrl' in brought ? brought.sourceUrl : undefined;+ const type = startPageType(+ { title: brought.title, html: brought.html, url: sourceUrl },+ siteType,+ );const newPageType = {pageType: type.pageType,- detectedFrom: type.detected ? ('content' as const) : null,+ detectedFrom: type.detected ? (sourceUrl ? ('page' as const) : ('content' as const)) : null,picked: false,};setStart({@@ -297,6 +342,7 @@ export function NewContentDialog({title: brought.title,source: { kind: 'content' },bodyHtml: brought.html,+ sourceUrl,...newPageType,newPageType,choice: { kind: 'create' },@@ -312,14 +358,25 @@ export function NewContentDialog({if (!start) return;setOpening(true);try {+ // An existing page Start chose to optimize brings its live text; if it+ // can't be read, the draft starts with its title, and says so.+ let bodyHtml = start.bodyHtml;+ if (existing) {+ const read = await importContentPage(brandId, existing.url);+ if ('page' in read) bodyHtml = cleanHtml(read.page.html, { keepImages: true }).html;+ else {+ bodyHtml = undefined;+ toast.warning(t('start.pageUnread', { page: pagePath(existing.url) }));+ }+ }const id = await createContentDraft(brandId, {method: startMethod,title: startTitle,pageType: start.pageType,typePicked: start.picked,source: start.source,- bodyHtml: start.bodyHtml,- sourceUrl: existing?.url,+ bodyHtml,+ sourceUrl: existing?.url ?? start.sourceUrl,});router.push(contentEditorPath(id));} catch (err) {@@ -508,11 +565,13 @@ export function NewContentDialog({<Tabsvalue={from}onValueChange={(value) => {- setFrom(value as 'paste' | 'file');+ setFrom(value as ContentFrom);+ setUrlError(null);setContent(null);}}><TabsList>+ <TabsTrigger value="url">{t('content.url')}</TabsTrigger><TabsTrigger value="paste">{t('content.paste')}</TabsTrigger><TabsTrigger value="file">{t('content.file')}</TabsTrigger></TabsList>@@ -526,7 +585,7 @@ export function NewContentDialog({</p><p className="text-xs text-muted-foreground">{[- content.fileName,+ content.fileName ?? (content.sourceUrl && pagePath(content.sourceUrl)),t('content.summary', {headings: content.headings,characters: formatNumber(content.characters),@@ -552,6 +611,43 @@ export function NewContentDialog({<X className="h-4 w-4" /></Button></div>+ ) : from === 'url' ? (+ <div className="space-y-1.5">+ <form+ className="flex gap-2"+ onSubmit={(e) => {+ e.preventDefault();+ void readPage();+ }}+ >+ <Input+ value={pageUrl}+ onChange={(e) => {+ setPageUrl(e.target.value);+ setUrlError(null);+ }}+ placeholder={t('content.urlPlaceholder')}+ aria-label={t('content.url')}+ aria-invalid={urlError !== null}+ inputMode="url"+ autoComplete="off"+ disabled={reading}+ />+ <Button type="submit" variant="outline" disabled={reading || !pageUrl.trim()}>+ {reading && <Loader2 className="h-4 w-4 animate-spin" />}+ {t(reading ? 'content.importing' : 'content.import')}+ </Button>+ </form>+ {urlError ? (+ <p className="text-xs text-destructive" role="alert">+ {urlError.code === 'http_error' && urlError.status+ ? t('content.urlErrors.http_status', { status: urlError.status })+ : t(`content.urlErrors.${urlError.code}`)}+ </p>+ ) : (+ <p className="text-xs text-muted-foreground">{t('content.urlHint')}</p>+ )}+ </div>) : from === 'paste' ? (<Textareavalue={pastedText}diff --git a/web/src/lib/actions/content-drafts-start.test.ts b/web/src/lib/actions/content-drafts-start.test.tsindex 7df668816..a4bb60d9c 100644--- a/web/src/lib/actions/content-drafts-start.test.ts+++ b/web/src/lib/actions/content-drafts-start.test.ts@@ -26,7 +26,10 @@ let rpcRows: unknown[];vi.mock('@/lib/action-center/assignees', () => ({ readAssignees: vi.fn(async () => new Map()) }));vi.mock('@/lib/supabase/server', () => ({createClient: vi.fn(async () => ({- auth: { getUser: async () => ({ data: { user: { id: 'u1' } } }) },+ auth: {+ getUser: async () => ({ data: { user: { id: 'u1' } } }),+ getSession: async () => ({ data: { session: { access_token: 'jwt' } } }),+ },rpc: async (fn: string, args: unknown) => {rpcCalls.push({ fn, args });return { data: rpcRows, error: null };@@ -54,7 +57,7 @@ vi.mock('@/lib/supabase/server', () => ({})),}));-const { createContentDraft, getContentOverlap, getContentStartOptions } =+const { createContentDraft, getContentOverlap, getContentStartOptions, importContentPage } =await import('./content-drafts');const inserted = () => calls.find((call) => call.op === 'insert')?.value as Record<string, unknown>;@@ -340,3 +343,66 @@ describe('optimizing an existing page instead (overlap check)', () => {expect(calls).toEqual([]);});});++describe('importContentPage', () => {+ const answer = (status: number, body: unknown) =>+ vi.fn(async () => new Response(JSON.stringify(body), { status }));++ it('asks the API to read the page, with the member’s session', async () => {+ const fetchMock = answer(200, {+ url: 'https://northwind.example/pricing',+ title: 'Pricing',+ html: '<h1>Pricing</h1>',+ });+ vi.stubGlobal('fetch', fetchMock);+ await expect(importContentPage(BRAND, ' northwind.example/pricing ')).resolves.toEqual({+ page: {+ url: 'https://northwind.example/pricing',+ title: 'Pricing',+ html: '<h1>Pricing</h1>',+ },+ });+ const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];+ expect(url).toMatch(/\/api\/content-pages\/import$/);+ expect(init.headers).toMatchObject({ Authorization: 'Bearer jwt' });+ expect(JSON.parse(init.body as string)).toEqual({+ brandId: BRAND,+ url: 'northwind.example/pricing',+ });+ vi.unstubAllGlobals();+ });++ it('names why the page couldn’t be read, and an unknown answer as failed', async () => {+ vi.stubGlobal('fetch', answer(422, { code: 'http_error', status: 404 }));+ await expect(importContentPage(BRAND, 'https://northwind.example/x')).resolves.toEqual({+ error: 'http_error',+ status: 404,+ });+ vi.stubGlobal('fetch', answer(500, { code: 'something_new' }));+ await expect(importContentPage(BRAND, 'https://northwind.example/x')).resolves.toEqual({+ error: 'failed',+ });+ vi.spyOn(console, 'error').mockImplementation(() => {});+ vi.stubGlobal(+ 'fetch',+ vi.fn(async () => {+ throw new Error('ECONNREFUSED');+ }),+ );+ await expect(importContentPage(BRAND, 'https://northwind.example/x')).resolves.toEqual({+ error: 'failed',+ });+ vi.unstubAllGlobals();+ });++ it('asks for nothing with a malformed brand or an empty address', async () => {+ const fetchMock = vi.fn();+ vi.stubGlobal('fetch', fetchMock);+ await expect(importContentPage('nope', 'https://x.example')).resolves.toEqual({+ error: 'invalid_url',+ });+ await expect(importContentPage(BRAND, ' ')).resolves.toEqual({ error: 'invalid_url' });+ expect(fetchMock).not.toHaveBeenCalled();+ vi.unstubAllGlobals();+ });+});diff --git a/web/src/lib/actions/content-drafts.ts b/web/src/lib/actions/content-drafts.tsindex 45be1de54..80b013b01 100644--- a/web/src/lib/actions/content-drafts.ts+++ b/web/src/lib/actions/content-drafts.ts@@ -1,6 +1,7 @@'use server';import { z } from 'zod';+import { API_BASE_URL } from '@/config/api';import { createClient } from '@/lib/supabase/server';import { readAssignees } from '@/lib/action-center/assignees';import {@@ -13,7 +14,10 @@ import {type DraftStart,type DraftStartOptions,type DraftStatus,+ type ImportedPage,+ type PageImportError,type PageType,+ PAGE_IMPORT_ERRORS,} from '@/lib/content/drafts';import { escapeHtml } from '@/lib/content/attachments';import {@@ -265,6 +269,55 @@ async function promptsFor(return { promptIds: data.map((p) => p.id), opportunityId: null };}+/**+ * A page of the brand's, read for Optimize content › Page URL and for+ * Start's "Optimize this page instead" (CE-003): its main content, title and+ * schema, through the API's guarded fetch, which reads only the brand's own+ * domains and never a private address. A page that can't be read comes back+ * as the reason, not a throw: production hides a server action's errors.+ */+export async function importContentPage(+ brandId: string,+ url: string,+): Promise<{ page: ImportedPage } | { error: PageImportError; status?: number }> {+ if (!draftId.safeParse(brandId).success || !url.trim() || url.length > 2048) {+ return { error: 'invalid_url' };+ }+ const supabase = await createClient();+ const {+ data: { session },+ } = await supabase.auth.getSession();+ if (!session) return { error: 'failed' };+ try {+ const res = await fetch(`${API_BASE_URL}/api/content-pages/import`, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ Authorization: `Bearer ${session.access_token}`,+ },+ body: JSON.stringify({ brandId, url: url.trim() }),+ });+ const body = (await res.json().catch(() => ({}))) as Record<string, unknown>;+ if (!res.ok) {+ const code = body.code as PageImportError;+ return {+ error: (PAGE_IMPORT_ERRORS as readonly string[]).includes(code) ? code : 'failed',+ ...(typeof body.status === 'number' && { status: body.status }),+ };+ }+ return {+ page: {+ url: String(body.url),+ title: String(body.title ?? ''),+ html: String(body.html ?? ''),+ },+ };+ } catch (err) {+ console.error('Failed to import a page:', err);+ return { error: 'failed' };+ }+}+const overlapSource = startInput.shape.source;/**diff --git a/web/src/lib/content/drafts.ts b/web/src/lib/content/drafts.tsindex 00cd7d203..c45f4eb53 100644--- a/web/src/lib/content/drafts.ts+++ b/web/src/lib/content/drafts.ts@@ -124,6 +124,29 @@ export interface DraftStart {sourceUrl?: string;}+/** A page of the brand's brought in by its URL (CE-003): main content, title and schema. */+export interface ImportedPage {+ /** The address after redirects. */+ url: string;+ title: string;+ html: string;+}++/** Why a page couldn't be brought in, as the API names it ('failed' for anything else). */+export const PAGE_IMPORT_ERRORS = [+ 'off_site',+ 'invalid_url',+ 'blocked',+ 'not_html',+ 'http_error',+ 'too_large',+ 'timeout',+ 'unreachable',+ 'too_many_redirects',+ 'failed',+] as const;+export type PageImportError = (typeof PAGE_IMPORT_ERRORS)[number];+/** One of the brand's pages AI already cites for a new page's prompts (CE-015). */export interface OverlapPage {url: string;diff --git a/web/src/lib/content/start.ts b/web/src/lib/content/start.tsindex 3ace5ff4b..1df3936fe 100644--- a/web/src/lib/content/start.ts+++ b/web/src/lib/content/start.ts@@ -45,11 +45,11 @@ export function fileTitle(name: string): string {}/**- * The page type Start picks first (CE-015): the one the title or content- * shows, else a blog post, which every kind of site writes.+ * The page type Start picks first (CE-015): the one the page's address,+ * content or title shows, else a blog post, which every kind of site writes.*/export function startPageType(- page: { title: string; html?: string },+ page: { title: string; html?: string; url?: string },siteType: SiteType,): { pageType: PageType; detected: boolean } {const detected = detectPageType(page, siteType);