Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(content): read a saved draft's links once a day for Trust: working or broken, and their page's date (#1415)
Cihandb2510828edb
diff --git a/server/src/lib/link-check.js b/server/src/lib/link-check.jsnew file mode 100644index 000000000..712649f76--- /dev/null+++ b/server/src/lib/link-check.js@@ -0,0 +1,283 @@+import { createHash } from 'node:crypto';+import * as cheerio from 'cheerio';+import supabaseAdmin from '../config/supabase.js';+import { jsonLd, metaContent, typesOf } from './audit/signals/helpers.js';+import { linkHref } from './content-links.js';+import { PageFetchError, fetchPage } from './safe-fetch.js';++/**+ * The links of a saved Content Editor draft, read for Trust (Quality spec+ * §2.2): does each answer ("No broken links"), and how old is the page it+ * points to ("Fresh sources"). Each link is read through the guarded fetch+ * (safe-fetch.js: public addresses only, redirects checked, time and size+ * capped) at most once a day, whoever's draft has it (content_link_checks,+ * 00258); a few reads at a time across the server, and the same link once+ * while it is being read.+ */++/** The most links one draft has read. */+export const MAX_LINKS = 20;+/** How long a read holds: a day, or a few hours for one that wouldn't tell. */+export const CHECK_TTL_MS = 24 * 60 * 60 * 1000;+export const UNKNOWN_TTL_MS = 6 * 60 * 60 * 1000;+/** Reads are dropped after a week. */+const KEEP_MS = 7 * 24 * 60 * 60 * 1000;+/** A link that hasn't answered by then is unknown, not broken. */+const LINK_TIMEOUT_MS = 8_000;+/**+ * A link's page is read up to this much, the rest cut off (fetchPage's+ * truncate): its head and article come first, so its date is still read.+ */+export const LINK_MAX_BYTES = 1024 * 1024;+const PER_CALL = 4;+/** Links read at once across every draft. */+const MAX_READS = 8;+/** The longest link kept (content_link_checks.url). */+const MAX_URL_LENGTH = 2048;++/** Answers that say the page is gone. */+const GONE = new Set([404, 410, 451]);+/** Network errors that say the site is nowhere. */+const NOWHERE = new Set(['ENOTFOUND', 'ECONNREFUSED']);++/**+ * How a read reads as a link: 'ok' when it answers (a page, or a file such+ * as a PDF), 'broken' when the page is gone, the site is nowhere, or it is+ * on a private or local address no reader of the published page can open,+ * and 'unknown' when it wouldn't tell: too slow, a bot wall (400, 401, 403,+ * 429), a server or certificate error that may pass, a redirect loop that+ * wants a cookie, a port the fetch won't read.+ *+ * @param {unknown} err the fetch's error, or null when it read the page+ * @returns {'ok'|'broken'|'unknown'}+ */+export function linkStatus(err) {+ if (!err) return 'ok';+ if (!(err instanceof PageFetchError)) return 'unknown';+ switch (err.code) {+ case 'not_html':+ case 'too_large':+ return 'ok';+ case 'http_error':+ return GONE.has(err.status) ? 'broken' : 'unknown';+ case 'unreachable':+ return NOWHERE.has(err.errno) ? 'broken' : 'unknown';+ case 'blocked':+ return 'broken';+ default:+ return 'unknown';+ }+}++/** The entities whose date is the page's own, not a sidebar's or a comment's. */+const MAIN_TYPES = new Set([+ 'article',+ 'blogposting',+ 'newsarticle',+ 'techarticle',+ 'scholarlyarticle',+ 'report',+ 'webpage',+ 'howto',+ 'faqpage',+]);++/**+ * The page's own newest date: its article's JSON-LD dateModified (or+ * datePublished), else its WebPage's, else its article:modified_time (or+ * published_time), else the first <time datetime> in its article or main. A+ * date ahead of `now` is a page's mistake, not its age. Null when it gives+ * none.+ */+export function pageDate(html, now = Date.now()) {+ const $ = cheerio.load(html);+ const usable = (value) => {+ const t = typeof value === 'string' ? Date.parse(value) : NaN;+ return Number.isNaN(t) || t > now ? null : t;+ };+ const dated = (nodes) =>+ nodes.map((n) => usable(n.dateModified) ?? usable(n.datePublished)).filter((t) => t !== null);+ const nodes = jsonLd({ $ }).nodes.filter((n) => typesOf(n).some((t) => MAIN_TYPES.has(t)));+ // An article's date before its page's: a WebPage's may be the site's build.+ const articles = dated(nodes.filter((n) => !typesOf(n).every((t) => t === 'webpage')));+ const main = articles.length ? articles : dated(nodes);+ const found = main.length+ ? Math.max(...main)+ : (usable(metaContent({ $ }, 'article:modified_time')) ??+ usable(metaContent({ $ }, 'article:published_time')) ??+ usable($('article time[datetime], main time[datetime]').first().attr('datetime')));+ return found === null ? null : new Date(found).toISOString();+}++/** Path parts whose read could do something: unsubscribe, sign out, confirm. */+const SIDE_EFFECT_PART =+ /^(?:unsubscribe|logout|log-out|signout|sign-out|confirm|confirm-email|verify|verify-email|activate|magic-link|reset-password)$/i;+const SIDE_EFFECT_PARAM = /^(?:token|access_token|auth|key|sig|signature|code|otp)$/i;++/**+ * Whether a link is one the check reads: not a one-time or account link (a+ * whole path part such as /unsubscribe, or a token in its query), and short+ * enough to keep.+ */+export function readableLink(url) {+ if (url.length > MAX_URL_LENGTH) return false;+ const parsed = new URL(url);+ if (parsed.pathname.split('/').some((part) => SIDE_EFFECT_PART.test(part))) return false;+ return ![...parsed.searchParams.keys()].some((name) => SIDE_EFFECT_PARAM.test(name));+}++const hashOf = (url) => createHash('sha256').update(url).digest('hex');++/** Reads under way across the server: one link read once, MAX_READS at a time. */+const reading = new Map();+let active = 0;+const waiting = [];+async function inSlot(work) {+ while (active >= MAX_READS) await new Promise((resolve) => waiting.push(resolve));+ active += 1;+ try {+ return await work();+ } finally {+ active -= 1;+ waiting.shift()?.();+ }+}++async function readLink(url, fetcher, now) {+ try {+ const page = await fetcher(url, {+ timeoutMs: LINK_TIMEOUT_MS,+ maxBytes: LINK_MAX_BYTES,+ truncate: true,+ });+ return { url, status: 'ok', httpStatus: page.status, date: pageDate(page.html, now) };+ } catch (err) {+ return {+ url,+ status: linkStatus(err),+ httpStatus: err instanceof PageFetchError ? (err.status ?? null) : null,+ date: null,+ };+ }+}++function readOnce(url, fetcher, now) {+ let read = reading.get(url);+ if (!read) {+ read = inSlot(() => readLink(url, fetcher, now)).finally(() => reading.delete(url));+ reading.set(url, read);+ }+ return read;+}++let prunedAt = 0;++/** Links one member may have read in an hour; past it, the rest wait for the next. */+export const HOURLY_READS = 200;+const HOUR_MS = 60 * 60 * 1000;+const spent = new Map();++/**+ * Takes up to `wanted` reads from the member's hour, at once (no await in+ * between, so asks made together can't each spend the whole hour), and+ * says how many it took.+ */+export function reserveReads(userId, wanted, now = Date.now()) {+ for (const [id, hour] of spent) if (now - hour.from >= HOUR_MS) spent.delete(id);+ const hour = spent.get(userId) ?? { from: now, used: 0 };+ const taken = Math.max(0, Math.min(wanted, HOURLY_READS - hour.used));+ hour.used += taken;+ spent.set(userId, hour);+ return taken;+}++/**+ * Each link's status and date, keyed by the link as given. Links are kept+ * as the check keys them (no hash, no tracking parameters), http(s) only,+ * MAX_LINKS at most. A read from the last day (an 'unknown' one from the+ * last few hours) comes from content_link_checks; of the rest, as many as+ * `reserve` grants are read now and kept there, and a link over it isn't in+ * the answer. `skipped` are the links never read: not http(s), one-time or+ * account links, too long, past MAX_LINKS.+ *+ * @param {string[]} hrefs the draft's links, as written+ * @param {{ db?: typeof supabaseAdmin, fetcher?: typeof fetchPage, now?: number, reserve?: (wanted: number) => number, log?: { error: Function } }} [deps]+ * @returns {Promise<{ reads: Map<string, { status: 'ok'|'broken'|'unknown', httpStatus: number|null, date: string|null }>, skipped: string[] }>}+ */+export async function checkLinks(hrefs, deps = {}) {+ const {+ db = supabaseAdmin,+ fetcher = fetchPage,+ now = Date.now(),+ reserve = (wanted) => wanted,+ log = console,+ } = deps;+ const byKey = new Map();+ const skipped = [];+ for (const href of new Set(hrefs)) {+ const key = linkHref(href);+ if (!key || !readableLink(key) || (!byKey.has(key) && byKey.size >= MAX_LINKS)) {+ skipped.push(href);+ continue;+ }+ byKey.set(key, [...(byKey.get(key) ?? []), href]);+ }+ const keys = [...byKey.keys()];+ const reads = new Map();+ if (!keys.length) return { reads, skipped };++ const known = new Map();+ const { data: kept, error } = await db+ .from('content_link_checks')+ .select('url, status, http_status, page_date, checked_at')+ .in('url_hash', keys.map(hashOf))+ .gte('checked_at', new Date(now - CHECK_TTL_MS).toISOString());+ // Without the kept reads, the links are read again.+ if (error) log.error({ err: error }, 'content link checks not read');+ for (const row of kept ?? []) {+ if (row.status === 'unknown' && Date.parse(row.checked_at) < now - UNKNOWN_TTL_MS) continue;+ known.set(row.url, { status: row.status, httpStatus: row.http_status, date: row.page_date });+ }++ const missing = keys.filter((key) => !known.has(key));+ const toRead = missing.slice(0, reserve(missing.length));+ const read = [];+ for (let i = 0; i < toRead.length; i += PER_CALL) {+ read.push(+ ...(await Promise.all(toRead.slice(i, i + PER_CALL).map((u) => readOnce(u, fetcher, now)))),+ );+ }+ if (read.length) {+ const checkedAt = new Date(now).toISOString();+ const { error: keepError } = await db.from('content_link_checks').upsert(+ read.map((r) => ({+ url_hash: hashOf(r.url),+ url: r.url,+ status: r.status,+ http_status: r.httpStatus,+ page_date: r.date,+ checked_at: checkedAt,+ })),+ { onConflict: 'url_hash' },+ );+ // The reads stand without being kept: the next ask reads them again.+ if (keepError) log.error({ err: keepError }, 'content link checks not kept');+ for (const r of read)+ known.set(r.url, { status: r.status, httpStatus: r.httpStatus, date: r.date });+ }++ if (now - prunedAt > 60 * 60 * 1000) {+ prunedAt = now;+ const { error: pruneError } = await db+ .from('content_link_checks')+ .delete()+ .lt('checked_at', new Date(now - KEEP_MS).toISOString());+ if (pruneError) log.error({ err: pruneError }, 'old content link checks not dropped');+ }++ for (const [key, given] of byKey) {+ const result = known.get(key);+ if (result) for (const href of given) reads.set(href, result);+ }+ return { reads, skipped };+}diff --git a/server/src/lib/link-check.test.js b/server/src/lib/link-check.test.jsnew file mode 100644index 000000000..6f902f93f--- /dev/null+++ b/server/src/lib/link-check.test.js@@ -0,0 +1,326 @@+import { createHash } from 'node:crypto';+import http from 'node:http';+import { describe, expect, it, vi } from 'vitest';++// The real client needs the service key; each test passes its own.+vi.mock('../config/supabase.js', () => ({ default: {} }));++const { PageFetchError, fetchPage, publicOnlyLookup } = await import('./safe-fetch.js');+const {+ CHECK_TTL_MS,+ HOURLY_READS,+ LINK_MAX_BYTES,+ MAX_LINKS,+ UNKNOWN_TTL_MS,+ checkLinks,+ linkStatus,+ pageDate,+ readableLink,+ reserveReads,+} = await import('./link-check.js');++/**+ * A saved draft's links for Trust (Quality spec §2.2): each read once a day+ * across the server, as working, broken or unknown, with its page's own date.+ */++const NOW = Date.parse('2026-10-23T12:00:00Z');+const hashOf = (url) => createHash('sha256').update(url).digest('hex');++/** content_link_checks, as the server reads, writes and prunes it. */+function fakeDb(rows = [], { readError = null } = {}) {+ const upserts = [];+ const deletes = [];+ const db = {+ from: () => {+ const query = {};+ const chain = {+ select: () => chain,+ in: (column, values) => {+ query.in = values;+ return chain;+ },+ gte: async (column, value) =>+ readError+ ? { data: null, error: readError }+ : {+ data: rows.filter((r) => query.in.includes(hashOf(r.url)) && r.checked_at >= value),+ error: null,+ },+ upsert: async (values) => {+ upserts.push(...values);+ return { error: null };+ },+ delete: () => chain,+ lt: async (column, value) => {+ deletes.push(value);+ return { error: null };+ },+ };+ return chain;+ },+ };+ return { db, upserts, deletes };+}++const page = (url, html = '') => ({ url, status: 200, html });+const silent = { error: vi.fn() };+const ago = (ms) => new Date(NOW - ms).toISOString();++describe('linkStatus', () => {+ it('reads a page or a file as working', () => {+ expect(linkStatus(null)).toBe('ok');+ expect(linkStatus(new PageFetchError('not_html', 'a PDF'))).toBe('ok');+ expect(linkStatus(new PageFetchError('too_large', 'big'))).toBe('ok');+ });++ it('reads a gone page, or a site that is nowhere, as broken', () => {+ for (const status of [404, 410, 451]) {+ expect(linkStatus(new PageFetchError('http_error', 'x', { status }))).toBe('broken');+ }+ for (const errno of ['ENOTFOUND', 'ECONNREFUSED']) {+ expect(linkStatus(new PageFetchError('unreachable', 'x', { errno }))).toBe('broken');+ }+ // A private or local address (http://192.168.1.1/): no reader of the page can open it.+ expect(linkStatus(new PageFetchError('blocked', 'x'))).toBe('broken');+ });++ it("doesn't call what may pass, or what wouldn't tell, broken", () => {+ for (const status of [400, 401, 403, 405, 429, 500, 503, 999]) {+ expect(linkStatus(new PageFetchError('http_error', 'x', { status }))).toBe('unknown');+ }+ for (const errno of ['ECONNRESET', 'EAI_AGAIN', 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', undefined]) {+ expect(linkStatus(new PageFetchError('unreachable', 'x', { errno }))).toBe('unknown');+ }+ // invalid_url: a port the fetch won't read, which a reader may still open.+ for (const code of ['timeout', 'invalid_url', 'too_many_redirects']) {+ expect(linkStatus(new PageFetchError(code, 'x'))).toBe('unknown');+ }+ expect(linkStatus(new Error('boom'))).toBe('unknown');+ });+});++describe('pageDate', () => {+ it("takes the main entity's date, not a sidebar's or a comment's", () => {+ const html = `<script type="application/ld+json">{"@graph":[+ {"@type":"BlogPosting","datePublished":"2019-01-05","dateModified":"2019-03-01"},+ {"@type":"Comment","dateCreated":"2026-01-01","datePublished":"2026-01-01"}]}</script>+ <aside><time datetime="2026-09-01">Latest post</time></aside>`;+ expect(pageDate(html, NOW)).toBe('2019-03-01T00:00:00.000Z');+ });++ it("takes an article's date before its WebPage's, which may be the site's build", () => {+ const html = `<script type="application/ld+json">[+ {"@type":"WebPage","dateModified":"2026-10-01"},+ {"@type":"Article","datePublished":"2021-04-04"}]</script>`;+ expect(pageDate(html, NOW)).toBe('2021-04-04T00:00:00.000Z');+ expect(+ pageDate(+ '<script type="application/ld+json">{"@type":"WebPage","dateModified":"2026-10-01"}</script>',+ NOW,+ ),+ ).toBe('2026-10-01T00:00:00.000Z');+ });++ it("falls back to article meta, then to the article's first <time>", () => {+ expect(+ pageDate(+ '<meta property="article:modified_time" content="2024-02-02"><article><time datetime="2020-01-01">x</time></article>',+ NOW,+ ),+ ).toBe('2024-02-02T00:00:00.000Z');+ expect(+ pageDate(+ '<aside><time datetime="2026-09-01">x</time></aside><main><time datetime="2023-05-05">y</time></main>',+ NOW,+ ),+ ).toBe('2023-05-05T00:00:00.000Z');+ });++ it('gives none for a page without a date, or with one ahead of now', () => {+ expect(pageDate('<p>No date</p>', NOW)).toBeNull();+ expect(pageDate('<article><time datetime="2027-01-01">x</time></article>', NOW)).toBeNull();+ });+});++describe('readableLink', () => {+ it('leaves out one-time and account links', () => {+ expect(readableLink('https://a.example/blog/post')).toBe(true);+ expect(readableLink('https://a.example/unsubscribe?u=1')).toBe(false);+ expect(readableLink('https://a.example/auth/magic-link/abc')).toBe(false);+ expect(readableLink('https://a.example/x?token=abc')).toBe(false);+ expect(readableLink('https://a.example/confirm-email')).toBe(false);+ // Words in an article's address aren't account links.+ expect(readableLink('https://a.example/blog/how-to-verify-a-domain')).toBe(true);+ expect(readableLink(`https://a.example/${'x'.repeat(2100)}`)).toBe(false);+ });+});++describe('checkLinks', () => {+ it('reads each link once, a hash or tracking variant with it, and keeps the reads', async () => {+ const { db, upserts } = fakeDb();+ const fetcher = vi.fn(async (url, opts) => {+ expect(opts.maxBytes).toBe(LINK_MAX_BYTES);+ if (url.includes('gone')) throw new PageFetchError('http_error', '404', { status: 404 });+ return page(url, '<article><time datetime="2025-06-01">June</time></article>');+ });+ const { reads, skipped } = await checkLinks(+ [+ 'https://a.example/x',+ 'https://a.example/x#part',+ 'https://a.example/x?utm_source=mail',+ 'https://gone.example/',+ '/pricing',+ 'mailto:[email protected]',+ 'https://a.example/unsubscribe',+ ],+ { db, fetcher, now: NOW, log: silent },+ );+ expect(fetcher).toHaveBeenCalledTimes(2);+ expect(skipped).toEqual(['/pricing', 'mailto:[email protected]', 'https://a.example/unsubscribe']);+ const ok = { status: 'ok', httpStatus: 200, date: '2025-06-01T00:00:00.000Z' };+ expect(Object.fromEntries(reads)).toEqual({+ 'https://a.example/x': ok,+ 'https://a.example/x#part': ok,+ 'https://a.example/x?utm_source=mail': ok,+ 'https://gone.example/': { status: 'broken', httpStatus: 404, date: null },+ });+ expect(upserts.map((u) => [u.url_hash, u.url, u.status])).toEqual([+ [hashOf('https://a.example/x'), 'https://a.example/x', 'ok'],+ [hashOf('https://gone.example/'), 'https://gone.example/', 'broken'],+ ]);+ });++ it('reads the date in the head of a page over the cap, and calls it working', async () => {+ // A long article page (inline app state and all), its date in <head>.+ const head =+ '<html><head><meta property="article:modified_time" content="2026-09-14T08:00:00Z"></head><body>';+ const server = http.createServer((req, res) => {+ res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });+ res.end(`${head}<p>${'x'.repeat(LINK_MAX_BYTES)}</p></body></html>`);+ });+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));+ const port = String(server.address().port);+ // site.test is the local server; any other name goes through the real check.+ const lookup = (hostname, options, callback) => {+ const done = typeof options === 'function' ? options : callback;+ const all = typeof options === 'object' && options.all;+ if (hostname !== 'site.test') return publicOnlyLookup(hostname, options, callback);+ return all ? done(null, [{ address: '127.0.0.1', family: 4 }]) : done(null, '127.0.0.1', 4);+ };+ try {+ const url = `http://site.test:${port}/long-read`;+ const { reads } = await checkLinks([url], {+ db: fakeDb().db,+ fetcher: (address, opts) =>+ fetchPage(address, { ...opts, lookup, ports: ['', '80', '443', port] }),+ now: NOW,+ log: silent,+ });+ expect(reads.get(url)).toEqual({+ status: 'ok',+ httpStatus: 200,+ date: '2026-09-14T08:00:00.000Z',+ });+ } finally {+ await new Promise((resolve) => server.close(resolve));+ }+ });++ it("uses a read from the last day, and reads an 'unknown' one again after a few hours", async () => {+ const row = (url, status, age) => ({+ url,+ status,+ http_status: null,+ page_date: null,+ checked_at: ago(age),+ });+ const { db } = fakeDb([+ row('https://fresh.example/', 'ok', 60_000),+ row('https://walled.example/', 'unknown', UNKNOWN_TTL_MS + 60_000),+ row('https://old.example/', 'broken', CHECK_TTL_MS + 60_000),+ ]);+ const fetcher = vi.fn(async (url) => page(url));+ await checkLinks(+ ['https://fresh.example/', 'https://walled.example/', 'https://old.example/'],+ {+ db,+ fetcher,+ now: NOW,+ log: silent,+ },+ );+ expect(fetcher.mock.calls.map(([url]) => url)).toEqual([+ 'https://walled.example/',+ 'https://old.example/',+ ]);+ });++ it('reads the links again when the kept reads cannot be read', async () => {+ const { db } = fakeDb([], { readError: { message: 'URI too long' } });+ const log = { error: vi.fn() };+ const fetcher = vi.fn(async (url) => page(url));+ const { reads } = await checkLinks(['https://a.example/'], { db, fetcher, now: NOW, log });+ expect(reads.get('https://a.example/').status).toBe('ok');+ expect(log.error).toHaveBeenCalled();+ });++ it(`reads ${MAX_LINKS} links at most, and no more than it may`, async () => {+ const { db } = fakeDb();+ const fetcher = vi.fn(async (url) => page(url));+ const urls = Array.from({ length: MAX_LINKS + 5 }, (_, i) => `https://s${i}.example/`);+ let result = await checkLinks(urls, { db, fetcher, now: NOW, log: silent });+ expect(result.reads.size).toBe(MAX_LINKS);+ expect(result.skipped).toHaveLength(5);+ fetcher.mockClear();+ result = await checkLinks(['https://t1.example/', 'https://t2.example/'], {+ db,+ fetcher,+ now: NOW,+ reserve: (wanted) => Math.min(wanted, 1),+ log: silent,+ });+ expect(fetcher).toHaveBeenCalledTimes(1);+ expect(result.reads.size).toBe(1);+ });++ it('reads a link two drafts ask for at once only once', async () => {+ const { db } = fakeDb();+ let release;+ const fetcher = vi.fn(+ (url) =>+ new Promise((resolve) => {+ release = () => resolve(page(url));+ }),+ );+ const deps = { db, fetcher, now: NOW, log: silent };+ const first = checkLinks(['https://shared.example/'], deps);+ const second = checkLinks(['https://shared.example/#x'], deps);+ await vi.waitFor(() => expect(fetcher).toHaveBeenCalled());+ release();+ const [a, b] = await Promise.all([first, second]);+ expect(fetcher).toHaveBeenCalledTimes(1);+ expect(a.reads.get('https://shared.example/').status).toBe('ok');+ expect(b.reads.get('https://shared.example/#x').status).toBe('ok');+ });++ it('drops reads older than a week, once an hour', async () => {+ const { db, deletes } = fakeDb();+ const fetcher = async (url) => page(url);+ const later = NOW + 2 * 60 * 60 * 1000;+ await checkLinks(['https://a.example/'], { db, fetcher, now: later, log: silent });+ await checkLinks(['https://b.example/'], { db, fetcher, now: later + 60_000, log: silent });+ expect(deletes).toEqual([new Date(later - 7 * 24 * 60 * 60 * 1000).toISOString()]);+ });+});++describe('reserveReads', () => {+ it('gives each member HOURLY_READS an hour, taken at once', () => {+ expect(reserveReads('u-budget', 150, NOW)).toBe(150);+ // Asks made together share what is left.+ expect(reserveReads('u-budget', 40, NOW + 1)).toBe(40);+ expect(reserveReads('u-budget', 40, NOW + 2)).toBe(HOURLY_READS - 190);+ expect(reserveReads('u-budget', 5, NOW + 3)).toBe(0);+ expect(reserveReads('u-budget', 5, NOW + 61 * 60 * 1000)).toBe(5);+ });+});diff --git a/server/src/lib/safe-fetch.js b/server/src/lib/safe-fetch.jsindex d8f139e6e..b07f74946 100644--- a/server/src/lib/safe-fetch.js+++ b/server/src/lib/safe-fetch.js@@ -33,12 +33,14 @@ export class PageFetchError extends Error {/*** @param {'invalid_url'|'blocked'|'unreachable'|'timeout'|'too_large'|'not_html'|'http_error'|'too_many_redirects'} code* @param {string} message- * @param {{ status?: number }} [extra]+ * @param {{ status?: number, errno?: string }} [extra] errno: the network error's code+ * (ENOTFOUND, ECONNREFUSED, a TLS one…) when the page couldn't be reached*/constructor(code, message, extra = {}) {super(message);this.code = code;this.status = extra.status;+ this.errno = extra.errno;}}@@ -151,8 +153,11 @@ function decode(buffer, contentType) {}}-/** One request, no redirects followed: the response, or where it redirects to. */-function requestOnce(url, { signal, lookup, maxBytes }) {+/**+ * One request, no redirects followed: the response, or where it redirects to.+ * A body over `maxBytes` is refused, or with `truncate` cut off there.+ */+function requestOnce(url, { signal, lookup, maxBytes, truncate }) {return new Promise((resolve, reject) => {const client = url.protocol === 'https:' ? https : http;const req = client.request(@@ -170,19 +175,20 @@ function requestOnce(url, { signal, lookup, maxBytes }) {},(res) => {const status = res.statusCode ?? 0;+ // What isn't read is dropped, not drained: a large file stops here.if (status >= 300 && status < 400 && res.headers.location) {- res.resume();+ res.destroy();resolve({ redirect: res.headers.location, status });return;}if (status < 200 || status >= 300) {- res.resume();+ res.destroy();reject(new PageFetchError('http_error', `The page answered ${status}`, { status }));return;}const contentType = res.headers['content-type'] ?? '';if (!/^\s*(text\/html|application\/xhtml\+xml)\b/i.test(contentType)) {- res.resume();+ res.destroy();reject(new PageFetchError('not_html', `The address is ${contentType || 'not a page'}`));return;}@@ -194,10 +200,19 @@ function requestOnce(url, { signal, lookup, maxBytes }) {const chunks = [];let size = 0;+ let full = false;body.on('data', (chunk) => {+ if (full) return;size += chunk.length;if (size > maxBytes) {+ full = true;req.destroy();+ if (truncate) {+ // The start of the page, up to the cap: the rest is never read.+ chunks.push(chunk.subarray(0, chunk.length - (size - maxBytes)));+ resolve({ status, contentType, body: decode(Buffer.concat(chunks), contentType) });+ return;+ }reject(new PageFetchError('too_large', 'The page is too large to read'));return;}@@ -218,8 +233,10 @@ function requestOnce(url, { signal, lookup, maxBytes }) {* A web page's HTML, fetched safely (see above).** @param {string} address- * @param {{ timeoutMs?: number, maxBytes?: number, maxRedirects?: number, lookup?: Function, ports?: string[], signal?: AbortSignal }} [opts]+ * @param {{ timeoutMs?: number, maxBytes?: number, truncate?: boolean, maxRedirects?: number, lookup?: Function, ports?: string[], signal?: AbortSignal }} [opts]* `signal` ends the read early too (a run's Stop), as its timeout does.+ * `truncate` reads a page over `maxBytes` up to it, instead of refusing it+ * ('too_large'), for a caller that needs only its start.* `lookup` and `ports` replace the public-only lookup and the web ports;* tests use them to reach a local server.* @returns {Promise<{ url: string, status: number, html: string }>} url is the address after redirects@@ -228,6 +245,7 @@ export async function fetchPage(address, opts = {}) {const {timeoutMs = FETCH_TIMEOUT_MS,maxBytes = MAX_PAGE_BYTES,+ truncate = false,maxRedirects = MAX_REDIRECTS,lookup = publicOnlyLookup,ports = WEB_PORTS,@@ -240,7 +258,7 @@ export async function fetchPage(address, opts = {}) {for (let hops = 0; ; hops += 1) {let res;try {- res = await requestOnce(url, { signal, lookup, maxBytes });+ res = await requestOnce(url, { signal, lookup, maxBytes, truncate });} catch (err) {if (err instanceof PageFetchError) throw err;if (signal.aborted) throw new PageFetchError('timeout', 'The page took too long to answer');@@ -248,6 +266,7 @@ export async function fetchPage(address, opts = {}) {throw new PageFetchError('unreachable',`The page could not be reached (${err?.code ?? err?.message})`,+ { errno: err?.code },);}if (!res.redirect) return { url: url.toString(), status: res.status, html: res.body };diff --git a/server/src/lib/safe-fetch.test.js b/server/src/lib/safe-fetch.test.jsindex 9b8e3824d..6e20d81ce 100644--- a/server/src/lib/safe-fetch.test.js+++ b/server/src/lib/safe-fetch.test.js@@ -193,4 +193,15 @@ describe('fetchPage against a local site', () => {await expect(get('/big', { maxBytes: 1_000 })).rejects.toMatchObject({ code: 'too_large' });await expect(get('/slow', { timeoutMs: 50 })).rejects.toMatchObject({ code: 'timeout' });});++ it('reads the start of a page over the cap when asked to, instead of refusing it', async () => {+ await expect(get('/big', { maxBytes: 1_000, truncate: true })).resolves.toMatchObject({+ status: 200,+ html: 'x'.repeat(1_000),+ });+ // Under the cap, the whole page, as without it.+ await expect(get('/page', { truncate: true })).resolves.toMatchObject({+ html: '<h1>Pricing</h1><p>Plans ünd prices</p>',+ });+ });});diff --git a/server/src/middleware/rate-limiter.js b/server/src/middleware/rate-limiter.jsindex 6f132c801..033a2d6c2 100644--- a/server/src/middleware/rate-limiter.js+++ b/server/src/middleware/rate-limiter.js@@ -56,6 +56,21 @@ export const apiLimiter = rateLimit({},});+// A draft's link check (Content Editor Trust): the editor asks after its+// links change, a few times a minute at most. Each ask can read links on+// other sites, so it has its own bucket besides the API's.+export const linkCheckLimiter = rateLimit({+ windowMs: 15 * 60 * 1000,+ max: 60,+ keyGenerator: apiLimiterKey,+ standardHeaders: true,+ legacyHeaders: false,+ message: {+ success: false,+ message: 'Too many requests, please try again later.',+ },+});+// Stricter limit for public/unauthenticated endpoints: 30 requests per 15 minutesexport const publicLimiter = rateLimit({windowMs: 15 * 60 * 1000,diff --git a/server/src/routes/content-pages.js b/server/src/routes/content-pages.jsindex de5267e56..6312279a6 100644--- a/server/src/routes/content-pages.js+++ b/server/src/routes/content-pages.js@@ -1,10 +1,13 @@import { Router } from 'express';+import * as cheerio from 'cheerio';import { z } from 'zod';import supabaseAdmin from '../config/supabase.js';import { assertBrandAccess } from '../lib/access.js';import { onBrandDomain } from '../lib/audit/host.js';+import { checkLinks, reserveReads } from '../lib/link-check.js';import { MAX_CONTENT_LENGTH, extractPageContent } from '../lib/page-content.js';import { PageFetchError, fetchPage } from '../lib/safe-fetch.js';+import { linkCheckLimiter } from '../middleware/rate-limiter.js';/*** Pages brought into the Content Editor (CE-003): Optimize content › Page@@ -82,4 +85,48 @@ router.post('/import', async (req, res) => {}});+const linksBody = z.object({ draftId: z.guid() });++/**+ * The links of a saved draft, for Quality's Trust (Quality spec §2.2):+ * whether each answers and how old its page is (link-check.js). Any member+ * of the draft's brand may ask; the links are read from the draft as saved,+ * never from the request, through the same guarded fetch, at most once a+ * day each and HOURLY_READS an hour per member (link-check.js). `skipped`+ * are the links it never reads (relative, one-time or account links).+ */+router.post('/links', linkCheckLimiter, async (req, res) => {+ const parsed = linksBody.safeParse(req.body);+ if (!parsed.success) return res.status(400).json({ error: 'invalid', code: 'invalid' });+ try {+ const { data: draft, error } = await supabaseAdmin+ .from('content_drafts')+ .select('brand_id, body_html')+ .eq('id', parsed.data.draftId)+ .maybeSingle();+ if (error) throw error;+ if (!draft) return res.status(404).json({ error: 'Draft not found', code: 'not_found' });+ await assertBrandAccess(draft.brand_id, req.user.id);++ const $ = cheerio.load(draft.body_html ?? '');+ const hrefs = $('a[href]')+ .map((_, el) => $(el).attr('href'))+ .get();+ const { reads, skipped } = await checkLinks(hrefs, {+ reserve: (wanted) => reserveReads(req.user.id, wanted),+ log: req.log,+ });+ return res.json({+ links: [...reads].map(([href, read]) => ({ href, ...read })),+ skipped,+ });+ } catch (err) {+ if (err?.status === 403 || err?.status === 404) {+ return res.status(err.status).json({ error: err.message, code: 'forbidden' });+ }+ req.log.error({ err }, 'content link check error');+ return res.status(500).json({ error: 'Failed to check the links', code: 'failed' });+ }+});+export default router;diff --git a/server/src/routes/content-pages.test.js b/server/src/routes/content-pages.test.jsindex 65320b8b3..2712c3632 100644--- a/server/src/routes/content-pages.test.js+++ b/server/src/routes/content-pages.test.js@@ -11,6 +11,7 @@ const BRAND = '33333333-3333-3333-3333-333333333333';let org;let fetched;let fetchResult;+let draftRow;vi.mock('../config/supabase.js', () => ({default: {@@ -18,14 +19,17 @@ vi.mock('../config/supabase.js', () => ({const result =name === 'brands'? { data: { id: BRAND, organization_id: 'o1' }, error: null }- : {- data: [{ domain: 'https://www.Northwind.example/' }, { domain: 'northwind.shop' }],- error: null,- };+ : name === 'content_drafts'+ ? { data: draftRow, error: null }+ : {+ data: [{ domain: 'https://www.Northwind.example/' }, { domain: 'northwind.shop' }],+ error: null,+ };const chain = {select: () => chain,eq: () => chain,single: async () => result,+ maybeSingle: async () => result,then: (resolve) => resolve(result),};return chain;@@ -45,6 +49,12 @@ vi.mock('../lib/safe-fetch.js', async (importOriginal) => {};});+const checkLinks = vi.fn();+vi.mock('../lib/link-check.js', async (importOriginal) => ({+ ...(await importOriginal()),+ checkLinks: (...args) => checkLinks(...args),+}));+const { default: router } = await import('./content-pages.js');const { PageFetchError } = await import('../lib/safe-fetch.js');@@ -53,6 +63,15 @@ let baseUrl;beforeEach(async () => {org = 'o1';+ draftRow = {+ brand_id: BRAND,+ body_html:+ '<p><a href="https://a.example/">a</a> <a href="https://a.example/#x">again</a> <a href="/pricing">ours</a></p>',+ };+ checkLinks.mockReset().mockResolvedValue({+ reads: new Map([['https://a.example/', { status: 'ok', httpStatus: 200, date: null }]]),+ skipped: ['/pricing'],+ });fetched = [];fetchResult = {url: 'https://northwind.example/pricing',@@ -135,3 +154,39 @@ describe('POST /api/content-pages/import', () => {expect(fetched).toEqual([]);});});++const DRAFT = '2f1d1c52-0f7a-4c1e-9b8a-5d6e7f8a9b0c';+const checkDraftLinks = (body) =>+ fetch(`${baseUrl}/links`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });++describe('POST /api/content-pages/links', () => {+ it("reads the saved draft's links, never the request's, and answers with each one's read", async () => {+ const res = await checkDraftLinks({ draftId: DRAFT, urls: ['https://evil.example/'] });+ expect(res.status).toBe(200);+ await expect(res.json()).resolves.toEqual({+ links: [{ href: 'https://a.example/', status: 'ok', httpStatus: 200, date: null }],+ skipped: ['/pricing'],+ });+ expect(checkLinks).toHaveBeenCalledWith(+ ['https://a.example/', 'https://a.example/#x', '/pricing'],+ expect.objectContaining({ reserve: expect.any(Function) }),+ );+ });++ it('refuses another organization’s draft, one that isn’t there, and an id that isn’t one', async () => {+ org = 'o2';+ let res = await checkDraftLinks({ draftId: DRAFT });+ expect(res.status).toBe(403);+ org = 'o1';+ draftRow = null;+ res = await checkDraftLinks({ draftId: DRAFT });+ expect(res.status).toBe(404);+ res = await checkDraftLinks({ draftId: 'nope' });+ expect(res.status).toBe(400);+ expect(checkLinks).not.toHaveBeenCalled();+ });+});diff --git a/supabase/migrations/00258_content_link_checks.sql b/supabase/migrations/00258_content_link_checks.sqlnew file mode 100644index 000000000..5a44d7165--- /dev/null+++ b/supabase/migrations/00258_content_link_checks.sql@@ -0,0 +1,38 @@+-- ============================================================================+-- 00258: the pages a Content Editor draft links to, read once a day+-- (Content Editor Quality, Trust)+-- ============================================================================+-- Trust's "No broken links" and "Fresh sources (≤ 24 months)" (Quality spec+-- §2.2) read each link a saved draft has: does it answer, and how old is the+-- page it points to. The server reads a link at most once a day (an+-- 'unknown' answer again after a few hours), whoever's draft has it; this is+-- that read. It holds only what any visitor of the linked page sees, so it+-- is shared, and only the server touches it. Rows go after a week.+--+-- * url_hash: sha256 of the link as the check keys it (no hash, no+-- tracking parameters); url is that link.+-- * status: 'ok' (it answers, a page or a file), 'broken' (gone, or the+-- site is nowhere), 'unknown' (it wouldn't tell: slow, a bot wall, a+-- server or certificate error that may pass).+-- * page_date: the page's own newest date (its main JSON-LD entity, else+-- its article:* meta, else the first <time> in its article), or null.++create table if not exists public.content_link_checks (+ url_hash text primary key check (char_length(url_hash) = 64),+ url text not null check (char_length(url) <= 2048),+ status text not null check (status in ('ok', 'broken', 'unknown')),+ http_status integer,+ page_date timestamptz,+ checked_at timestamptz not null default now()+);++create index if not exists content_link_checks_checked_at+ on public.content_link_checks (checked_at);++-- Service-role only, no policy.+alter table public.content_link_checks enable row level security;+revoke all on table public.content_link_checks from anon, authenticated;+grant all on table public.content_link_checks to service_role;++comment on table public.content_link_checks is+ 'Content Editor Quality (Trust): each linked page''s last read — answers or not, and its own date. Server only; read at most once a day, kept a week.';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex b2e62b2a7..e00e87e37 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -37882,3 +37882,45 @@ create trigger content_drafts_authorbefore insert or update of author_id on public.content_draftsfor each row execute function public.content_draft_author();+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00258_content_link_checks.sql+-- ─────────────────────────────────────────────────────────────────────────+-- ============================================================================+-- 00258: the pages a Content Editor draft links to, read once a day+-- (Content Editor Quality, Trust)+-- ============================================================================+-- Trust's "No broken links" and "Fresh sources (≤ 24 months)" (Quality spec+-- §2.2) read each link a saved draft has: does it answer, and how old is the+-- page it points to. The server reads a link at most once a day (an+-- 'unknown' answer again after a few hours), whoever's draft has it; this is+-- that read. It holds only what any visitor of the linked page sees, so it+-- is shared, and only the server touches it. Rows go after a week.+--+-- * url_hash: sha256 of the link as the check keys it (no hash, no+-- tracking parameters); url is that link.+-- * status: 'ok' (it answers, a page or a file), 'broken' (gone, or the+-- site is nowhere), 'unknown' (it wouldn't tell: slow, a bot wall, a+-- server or certificate error that may pass).+-- * page_date: the page's own newest date (its main JSON-LD entity, else+-- its article:* meta, else the first <time> in its article), or null.++create table if not exists public.content_link_checks (+ url_hash text primary key check (char_length(url_hash) = 64),+ url text not null check (char_length(url) <= 2048),+ status text not null check (status in ('ok', 'broken', 'unknown')),+ http_status integer,+ page_date timestamptz,+ checked_at timestamptz not null default now()+);++create index if not exists content_link_checks_checked_at+ on public.content_link_checks (checked_at);++-- Service-role only, no policy.+alter table public.content_link_checks enable row level security;+revoke all on table public.content_link_checks from anon, authenticated;+grant all on table public.content_link_checks to service_role;++comment on table public.content_link_checks is+ 'Content Editor Quality (Trust): each linked page''s last read — answers or not, and its own date. Server only; read at most once a day, kept a week.';+