Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.
TypeScript aeoai-agentai-searchanswer-engine-optimizationbrand-monitoringchatgptclaudecopilotdockergeminigeogoogle-ai-overviewsgrokmcpnextjsopen-sourceperplexityseosupabasetypescript
feat(audit): read a brand's site trust: About, Contact, editorial policy, Organization schema and author pages (CE-043) (#1417)
Cihaned347ae8b6be
diff --git a/server/src/lib/site-trust.js b/server/src/lib/site-trust.jsnew file mode 100644index 000000000..3c8dea8dd--- /dev/null+++ b/server/src/lib/site-trust.js@@ -0,0 +1,452 @@+import * as cheerio from 'cheerio';+import supabaseAdmin from '../config/supabase.js';+import { fetchViaScrapeDo } from './audit/fetcher.js';+import { jsonLd, typesOf } from './audit/signals/helpers.js';+import { PageFetchError, fetchPage } from './safe-fetch.js';+import { looksBlocked } from './site-pages.js';++/**+ * A brand's site trust (Content Editor Quality spec §8, CE-043): five+ * site-level checks Site Audit shows as "Site trust {n}/5", and Trust's 5+ * points on every page. Read from the brand's page list (site_pages), its+ * home page and its authors (Settings › Authors), at most once a day or when+ * someone asks again (not more than every few minutes), and kept in+ * site_trust (00259). Only the brand's own site is read. A read a site+ * didn't answer all of (slow, unreachable, a server error, a bot wall) is+ * kept only a few minutes. A home page behind a bot wall is read through+ * Scrape.do, unrendered, as the page list reads one (site-pages.js): at most+ * once a day per brand.+ */++export const SITE_TRUST_CHECKS = [+ 'about',+ 'contact',+ 'editorial',+ 'organizationSchema',+ 'authorPages',+];+/** How long a read holds, and how soon asking again (or a partial read) reads again. */+export const SITE_TRUST_TTL_MS = 24 * 60 * 60 * 1000;+export const RECHECK_MS = 10 * 60 * 1000;+const FETCH_TIMEOUT_MS = 8_000;+const FETCH_MAX_BYTES = 1024 * 1024;+/** The authors whose page is looked for at the usual addresses, when the page list has none. */+export const MAX_AUTHOR_PROBES = 3;+/** The usual addresses of an author's page: WordPress's, and the common plural. */+const AUTHOR_PROBES = ['/author/', '/authors/'];++/** Between a title's words and the site's name: "About us · Acme". */+const SEPARATOR = String.raw`(?:\s*[|·•–—:-]|$)`;++/**+ * The pages three checks look for: the last part of the address, in the+ * languages the app reads, or a title that is just that ("About us · Acme",+ * not "About our pricing"); the addresses tried on the site when the page+ * list has none, whose title or H1 must have the word.+ */+const PAGES = {+ about: {+ part: /^(?:about|about-us|company|who-we-are|our-story|hakkimizda|hakkımızda|uber-uns|ueber-uns|a-propos|sobre-nosotros|quienes-somos)$/i,+ title: new RegExp(+ `^(?:about(?:\\s+us)?|who we are|hakkımızda|über uns|à propos|sobre nosotros|quiénes somos)${SEPARATOR}`,+ 'i',+ ),+ probe: ['/about', '/about-us'],+ word: /\babout\b|hakkımızda|über uns|à propos|sobre nosotros|quiénes somos/i,+ },+ contact: {+ part: /^(?:contact|contact-us|contacts|iletisim|iletişim|kontakt|contacto|nous-contacter)$/i,+ title: new RegExp(+ `^(?:contact(?:\\s+us)?|iletişim|kontakt|contacto|nous contacter)${SEPARATOR}`,+ 'i',+ ),+ probe: ['/contact', '/contact-us'],+ word: /\bcontact|iletişim|kontakt|contacto/i,+ },+ editorial: {+ part: /^(?:editorial-policy|editorial-guidelines|editorial-standards|editorial-process|how-we-review|how-we-test|review-process|methodology|our-methodology|fact-checking-policy)$/i,+ title: new RegExp(+ `^(?:(?:our )?editorial (?:policy|guidelines|standards|process)|how we (?:review|test)|(?:our )?methodology|fact[- ]checking policy)${SEPARATOR}`,+ 'i',+ ),+ probe: ['/editorial-policy', '/how-we-review'],+ word: /editorial|how we (?:review|test)|methodology|fact[- ]check/i,+ },+};++/** The folders an author's page sits in: "/authors/ada-lovelace". */+const AUTHOR_FOLDERS =+ /^(?:author|authors|team|people|staff|writers|contributors|yazar|yazarlar|autor|autoren|auteur|auteurs)$/i;++/** Organization and its kinds, as JSON-LD names them (the commonest local ones too). */+const ORGANIZATION_TYPES = new Set([+ 'organization',+ 'corporation',+ 'store',+ 'ngo',+ 'professionalservice',+ 'restaurant',+ 'cafeorcoffeeshop',+ 'bakery',+ 'dentist',+ 'physician',+ 'hotel',+ 'legalservice',+ 'attorney',+ 'realestateagent',+ 'autodealer',+ 'medicalclinic',+ 'healthclub',+ 'beautysalon',+ 'travelagency',+ 'insuranceagency',+ 'accountingservice',+ 'financialservice',+ 'bankorcreditunion',+ 'collegeoruniversity',+]);++/** A JSON-LD type without its vocabulary ("https://schema.org/Organization" → "organization"). */+const bareType = (type) => type.replace(/^.*[/:]/, '').toLowerCase();+const isOrganizationType = (type) => {+ const t = bareType(type);+ return ORGANIZATION_TYPES.has(t) || t.endsWith('organization') || t.endsWith('business');+};++const pathParts = (url) => {+ try {+ return new URL(url).pathname.split('/').filter(Boolean).map(decodeURIComponent);+ } catch {+ return [];+ }+};++/** Whether a page of the list is the one a check looks for: its address's last part, or its title. */+function isPage(p, kind) {+ const { part, title } = PAGES[kind];+ const last = pathParts(p.url).at(-1);+ return (!!last && part.test(last)) || title.test((p.title ?? '').trim());+}++/** The page of the list a check looks for, or null. */+export function findPage(pages, kind) {+ return pages.find((p) => isPage(p, kind))?.url ?? null;+}++/** Whether a page's JSON-LD names an Organization (or a kind of one). */+export function hasOrganization(html) {+ return jsonLd({ $: cheerio.load(html) }).nodes.some((n) => typesOf(n).some(isOrganizationType));+}++/** "Ada Lovelace" → "ada-lovelace": how an author's page address names them. */+export function nameSlug(name) {+ return name+ .normalize('NFKD')+ .replace(/[̀-ͯ]/g, '')+ .replace(/ı/g, 'i')+ .toLowerCase()+ .replace(/[^a-z0-9]+/g, '-')+ .replace(/^-|-$/g, '');+}++const escapeRegExp = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');++/**+ * An author's page on the site: one of the list's in an author folder named+ * for them ("/authors/ada-lovelace"), or titled just their name ("Ada+ * Lovelace · Acme"), or a profile link of theirs on the brand's own site.+ */+export function authorPage(pages, author, onSite) {+ const slug = nameSlug(author.name);+ const title = new RegExp(`^${escapeRegExp(author.name.trim())}${SEPARATOR}`, 'i');+ const found = pages.find((p) => {+ const parts = pathParts(p.url);+ const named = parts.findIndex((x) => x.toLowerCase() === slug);+ return (+ (!!slug && named > 0 && AUTHOR_FOLDERS.test(parts[named - 1])) ||+ title.test((p.title ?? '').trim())+ );+ });+ if (found) return found.url;+ return (author.same_as ?? []).find((link) => onSite(link)) ?? null;+}++const bareHost = (host) => host.toLowerCase().replace(/^www\./, '');++/** The brand's main domain, bare ("acme.com"), or null. */+function mainDomain(domains) {+ const primary = domains.find((d) => d.is_primary) ?? domains[0];+ if (!primary) return null;+ return bareHost(primary.domain.replace(/^https?:\/\//, '').split('/')[0]);+}++/**+ * Whether a failed read says nothing yet: slow, unreachable, a server error,+ * or a bot wall (401, 403, 429), which may hide a page the site has.+ */+const unsure = (err) =>+ !(err instanceof PageFetchError) ||+ err.code === 'timeout' ||+ err.code === 'unreachable' ||+ (err.code === 'http_error' && ((err.status ?? 0) >= 500 || looksBlocked(err.status, '')));++/** Whether a failed read was turned away by a bot wall, so Scrape.do may try. */+const walledError = (err) =>+ err instanceof PageFetchError && err.code === 'http_error' && looksBlocked(err.status, '');++/** A page's title and H1, to tell it from the home page. */+function heads(html) {+ const $ = cheerio.load(html);+ return { title: $('title').first().text().trim(), h1: $('h1').first().text().trim() };+}++/**+ * Whether a page that answered is a bot wall's challenge ("Just a moment…",+ * "Access denied"), by its title and H1: the rest of a real page may name a+ * captcha (a contact form's script) without being one.+ */+function walledPage(html) {+ const { title, h1 } = heads(html);+ return looksBlocked(200, `${title} ${h1}`);+}++/** Whether a title or H1 names the author, as whole words ("Ada Lovelace, Author at Acme"). */+function namesAuthor(text, name) {+ const words = escapeRegExp(name.trim()).replace(/\s+/g, '\\s+');+ return new RegExp(`(?<![\\p{L}\\p{N}])${words}(?![\\p{L}\\p{N}])`, 'iu').test(text);+}++/** The home page through Scrape.do, unrendered: one request, one credit. */+const scrapeHome = (url) => fetchViaScrapeDo(url, { render: false, retries: 0 });++/**+ * Reads the five checks for a brand: each with whether it passes and the page+ * that passes it, whether the site answered all of it (`complete`), and+ * whether its home page went through Scrape.do (`scraped`).+ *+ * `scraper` reads a home page a bot wall turned away (null: none is read);+ * `keptHome` is the kept read's Organization check while a Scrape.do read+ * from today stands: a pass stands while the wall hides the home page.+ */+export async function computeSiteTrust(+ brandId,+ { db = supabaseAdmin, fetcher = fetchPage, scraper = null, keptHome = null } = {},+) {+ const { data: brand, error: brandError } = await db+ .from('brands')+ .select('organization_id')+ .eq('id', brandId)+ .single();+ if (brandError) throw new Error(brandError.message);+ const [domainsRes, pagesRes, authorsRes, draftsRes, defaultRes] = await Promise.all([+ db.from('brand_domains').select('domain, is_primary').eq('brand_id', brandId),+ db.from('site_pages').select('url, title, fetch_status').eq('brand_id', brandId).order('url'),+ db+ .from('content_authors')+ .select('id, name, same_as')+ .eq('organization_id', brand.organization_id)+ .order('name'),+ db+ .from('content_drafts')+ .select('author_id')+ .eq('brand_id', brandId)+ .not('author_id', 'is', null),+ db.from('content_author_defaults').select('author_id').eq('brand_id', brandId),+ ]);+ for (const res of [domainsRes, pagesRes, authorsRes, draftsRes, defaultRes]) {+ if (res.error) throw new Error(res.error.message);+ }+ const domain = mainDomain(domainsRes.data ?? []);+ // A page that answered with an error isn't one the site has.+ const pages = (pagesRes.data ?? []).filter(+ (p) => p.fetch_status == null || (p.fetch_status > 0 && p.fetch_status < 400),+ );+ const onSite = (link) => {+ try {+ const host = bareHost(new URL(link).hostname);+ return !!domain && (host === domain || host.endsWith(`.${domain}`));+ } catch {+ return false;+ }+ };+ let complete = true;+ /** A page of the site, or why not: `walled` when a bot wall turned the read away. */+ const attempt = async (url) => {+ try {+ const page = await fetcher(url, { timeoutMs: FETCH_TIMEOUT_MS, maxBytes: FETCH_MAX_BYTES });+ // Only the brand's own site: a page that lands elsewhere isn't read.+ if (!onSite(page.url)) return { page: null };+ return walledPage(page.html) ? { page: null, walled: true } : { page };+ } catch (err) {+ return { page: null, walled: walledError(err), unsure: unsure(err) };+ }+ };+ const read = async (url) => {+ const { page, walled, unsure: notSure } = await attempt(url);+ if (walled || notSure) complete = false;+ return page;+ };++ // The home page first: a site that answers every address with it has none of the others there.+ const home = domain ? `https://${domain}/` : null;+ let homePage = null;+ let scraped = false;+ let homeCheck = null;+ if (home) {+ const direct = await attempt(home);+ homePage = direct.page;+ if (direct.walled && scraper) {+ // Turned away: read once through Scrape.do, as the page list does.+ scraped = true;+ const proxied = await scraper(home).catch(() => null);+ if (proxied?.ok && proxied.html && !walledPage(proxied.html)) {+ homePage = { url: home, html: proxied.html };+ }+ }+ if (!homePage && (direct.walled || direct.unsure)) {+ // What a Scrape.do read showed earlier today stands; else it isn't known yet.+ if (keptHome?.ok) homeCheck = { ok: true, url: home };+ else complete = false;+ }+ }+ const homeHeads = homePage ? heads(homePage.html) : null;++ // An address the site answers counts when it isn't the home page again, its+ // address or title is the page's ("/about" → "/company/about" too), and its+ // title or H1 says what it is.+ const probe = async (kind) => {+ for (const path of domain ? PAGES[kind].probe : []) {+ const page = await read(`https://${domain}${path}`);+ if (!page) continue;+ const { title, h1 } = heads(page.html);+ if (homeHeads && title === homeHeads.title && h1 === homeHeads.h1) continue;+ if (isPage({ url: page.url, title }, kind) && PAGES[kind].word.test(`${title} ${h1}`)) {+ return page.url;+ }+ }+ return null;+ };+ const found = await Promise.all(+ ['about', 'contact', 'editorial'].map(async (kind) => findPage(pages, kind) ?? probe(kind)),+ );+ const checks = {+ about: { ok: !!found[0], url: found[0] },+ contact: { ok: !!found[1], url: found[1] },+ editorial: { ok: !!found[2], url: found[2] },+ organizationSchema: homeCheck ?? {+ ok: !!homePage && hasOrganization(homePage.html),+ url: home,+ },+ };++ // The brand's authors: the ones its pages are by and its default; else the workspace's.+ const used = new Set([+ ...(draftsRes.data ?? []).map((d) => d.author_id),+ ...(defaultRes.data ?? []).map((d) => d.author_id),+ ]);+ const all = authorsRes.data ?? [];+ const brandAuthors = all.some((a) => used.has(a.id)) ? all.filter((a) => used.has(a.id)) : all;+ const authors = brandAuthors.map((a) => ({ name: a.name, url: authorPage(pages, a, onSite) }));+ // An author the list has no page for: their page at the usual addresses,+ // with the probes' guards (not the home page again, on the site), titled or+ // headed with their name. A few authors only, so a read stays bounded.+ const probeAuthor = async (author) => {+ const slug = nameSlug(author.name);+ for (const folder of slug ? AUTHOR_PROBES : []) {+ const page = await read(`https://${domain}${folder}${slug}`);+ if (!page) continue;+ const { title, h1 } = heads(page.html);+ if (homeHeads && title === homeHeads.title && h1 === homeHeads.h1) continue;+ if (namesAuthor(`${title} ${h1}`, author.name)) return page.url;+ }+ return null;+ };+ const unmatched = domain ? authors.filter((a) => !a.url).slice(0, MAX_AUTHOR_PROBES) : [];+ await Promise.all(+ unmatched.map(async (a) => {+ a.url = await probeAuthor(a);+ }),+ );+ checks.authorPages = { ok: authors.length > 0 && authors.every((a) => a.url), authors };++ return {+ checks,+ score: SITE_TRUST_CHECKS.filter((k) => checks[k].ok).length,+ complete,+ scraped,+ };+}++/** Reads under way, by brand: two asks at once read once. */+const reading = new Map();++/**+ * The brand's site trust: the kept read while it holds (a day; a few minutes+ * when asked again or when the site didn't answer all of it), else a new+ * read, kept. `complete` is false when the site didn't answer all of it: the+ * checks it couldn't read are read again soon, not to be fixed.+ *+ * A home page behind a bot wall goes through Scrape.do once a day per brand+ * (site_trust.scraped_at), whoever asks and however often: a read within the+ * day keeps what that one showed.+ *+ * @returns {Promise<{ checks: object, score: number, complete: boolean, checkedAt: string }>}+ */+export async function readSiteTrust(+ brandId,+ {+ refresh = false,+ db = supabaseAdmin,+ fetcher = fetchPage,+ scraper = process.env.SCRAPEDO_API_KEY ? scrapeHome : null,+ now = Date.now(),+ } = {},+) {+ const { data: kept, error } = await db+ .from('site_trust')+ .select('checks, score, complete, checked_at, scraped_at')+ .eq('brand_id', brandId)+ .maybeSingle();+ if (error) throw new Error(error.message);+ if (kept) {+ const age = now - Date.parse(kept.checked_at);+ const holds = refresh || kept.complete === false ? RECHECK_MS : SITE_TRUST_TTL_MS;+ if (age < holds) {+ return {+ checks: kept.checks,+ score: kept.score,+ complete: kept.complete !== false,+ checkedAt: kept.checked_at,+ };+ }+ }+ let pending = reading.get(brandId);+ if (!pending) {+ pending = (async () => {+ const scrapedToday =+ !!kept?.scraped_at && now - Date.parse(kept.scraped_at) < SITE_TRUST_TTL_MS;+ const { checks, score, complete, scraped } = await computeSiteTrust(brandId, {+ db,+ fetcher,+ scraper: scrapedToday ? null : scraper,+ keptHome: scrapedToday ? (kept.checks?.organizationSchema ?? null) : null,+ });+ const checkedAt = new Date(now).toISOString();+ const { error: keepError } = await db.from('site_trust').upsert(+ {+ brand_id: brandId,+ checks,+ score,+ complete,+ checked_at: checkedAt,+ scraped_at: scraped ? checkedAt : (kept?.scraped_at ?? null),+ },+ { onConflict: 'brand_id' },+ );+ if (keepError) throw new Error(keepError.message);+ return { checks, score, complete, checkedAt };+ })().finally(() => reading.delete(brandId));+ reading.set(brandId, pending);+ }+ return pending;+}diff --git a/server/src/lib/site-trust.test.js b/server/src/lib/site-trust.test.jsnew file mode 100644index 000000000..cb8ea0361--- /dev/null+++ b/server/src/lib/site-trust.test.js@@ -0,0 +1,453 @@+import { describe, expect, it, vi } from 'vitest';++// The real client needs the service key; each test passes its own.+vi.mock('../config/supabase.js', () => ({ default: {} }));++const { PageFetchError } = await import('./safe-fetch.js');+const {+ MAX_AUTHOR_PROBES,+ RECHECK_MS,+ SITE_TRUST_TTL_MS,+ authorPage,+ computeSiteTrust,+ findPage,+ hasOrganization,+ nameSlug,+ readSiteTrust,+} = await import('./site-trust.js');++/**+ * Site trust (Quality spec §8, CE-043): About, Contact, an editorial policy,+ * Organization schema on the home page, and a page for each author, read+ * from the brand's own site.+ */++const NOW = Date.parse('2026-10-23T12:00:00Z');+const page = (url, title = null, fetch_status = 200) => ({ url, title, fetch_status });+const html = (title, h1 = '', extra = '') => `<title>${title}</title><h1>${h1}</h1>${extra}`;+const ORG = '<script type="application/ld+json">{"@type":"Organization"}</script>';++/** The tables computeSiteTrust and readSiteTrust read, and the writes. */+function fakeDb(tables) {+ const writes = [];+ const db = {+ from: (table) => {+ const chain = {+ select: () => chain,+ eq: () => chain,+ not: () => chain,+ order: () => chain,+ single: async () => ({ data: tables[table], error: null }),+ maybeSingle: async () => ({ data: tables[table] ?? null, error: null }),+ then: (resolve) => resolve({ data: tables[table] ?? [], error: null }),+ upsert: async (row) => {+ writes.push({ table, row });+ return { error: null };+ },+ };+ return chain;+ },+ };+ return { db, writes };+}++describe('findPage', () => {+ const pages = [+ page('https://acme.com/blog/about-our-pricing', 'About our pricing changes'),+ page('https://acme.com/company/news/launch', 'We launched'),+ page('https://acme.com/company/', 'Our company'),+ page('https://acme.com/help', 'Contact us · Acme'),+ ];++ it("finds a page by its address's last part, or a title that is just that", () => {+ expect(findPage(pages, 'about')).toBe('https://acme.com/company/');+ expect(findPage(pages, 'contact')).toBe('https://acme.com/help');+ expect(findPage([page('https://acme.com/how-we-review')], 'editorial')).toBe(+ 'https://acme.com/how-we-review',+ );+ expect(findPage([page('https://acme.com/p/12', 'Editorial policy | Acme')], 'editorial')).toBe(+ 'https://acme.com/p/12',+ );+ });++ it('finds none in posts that only mention the word', () => {+ expect(findPage(pages.slice(0, 2), 'about')).toBeNull();+ expect(+ findPage([page('https://acme.com/blog/x', 'How to write an editorial policy')], 'editorial'),+ ).toBeNull();+ });+});++describe('hasOrganization', () => {+ const ld = (type) =>+ `<script type="application/ld+json">${JSON.stringify({ '@type': type })}</script>`;+ it('reads Organization, a kind of it, or one with its vocabulary', () => {+ expect(hasOrganization(ld('Organization'))).toBe(true);+ expect(hasOrganization(ld('Restaurant'))).toBe(true);+ expect(hasOrganization(ld('LocalBusiness'))).toBe(true);+ expect(hasOrganization(ld('https://schema.org/Organization'))).toBe(true);+ expect(hasOrganization(ld('WebSite'))).toBe(false);+ });+});++describe('authorPage', () => {+ const onSite = (link) => link.startsWith('https://acme.com/');+ const author = (name, same_as = []) => ({ name, same_as });++ it('finds the page in an author folder named for them, or titled just their name', () => {+ const pages = [+ page('https://acme.com/authors/ada-lovelace'),+ page('https://acme.com/blog/ada-lovelace'),+ page('https://acme.com/x', 'Bob Stone · Acme'),+ ];+ expect(authorPage(pages, author('Ada Lovelace'), onSite)).toBe(+ 'https://acme.com/authors/ada-lovelace',+ );+ expect(authorPage(pages, author('Bob Stone'), onSite)).toBe('https://acme.com/x');+ });++ it("doesn't take a post about them, or a name inside another word", () => {+ const pages = [+ page('https://acme.com/blog/ada-lovelace', "Ada Lovelace's legacy"),+ page('https://acme.com/guides/california', 'California guide'),+ ];+ expect(authorPage(pages, author('Ada Lovelace'), onSite)).toBeNull();+ expect(authorPage(pages, author('Cal'), onSite)).toBeNull();+ });++ it('takes a profile link of theirs on the brand’s site', () => {+ expect(+ authorPage([], author('Cal', ['https://x.com/cal', 'https://acme.com/cal']), onSite),+ ).toBe('https://acme.com/cal');+ expect(authorPage([], author('Dee', ['https://x.com/dee']), onSite)).toBeNull();+ });++ it('names an author as an address would', () => {+ expect(nameSlug('Çağrı Işık')).toBe('cagri-isik');+ expect(nameSlug('José García')).toBe('jose-garcia');+ });+});++describe('computeSiteTrust', () => {+ const tables = {+ brands: { organization_id: 'o1' },+ brand_domains: [+ { domain: 'https://www.acme.com/', is_primary: true },+ { domain: 'acme.de', is_primary: false },+ ],+ site_pages: [+ page('https://acme.com/about-us', 'About us'),+ page('https://acme.com/authors/ada-lovelace'),+ page('https://acme.com/contact', 'Contact', 404),+ ],+ content_authors: [+ { id: 'a1', name: 'Ada Lovelace', same_as: [] },+ { id: 'a2', name: 'Bob Stone', same_as: [] },+ { id: 'a3', name: 'Other Brand Writer', same_as: [] },+ ],+ content_drafts: [{ author_id: 'a1' }],+ content_author_defaults: [{ author_id: 'a2' }],+ };+ const HOME = html('Acme — coffee for teams', 'Coffee for teams', ORG);++ it('reads the list, tries the usual addresses, and the home page', async () => {+ const { db } = fakeDb(tables);+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') return { url, html: HOME };+ if (url === 'https://acme.com/contact') return { url, html: html('Contact – Acme') };+ // A site that answers every address with its home page: no editorial policy there.+ return { url, html: HOME };+ });+ const { checks, score, complete } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.about).toEqual({ ok: true, url: 'https://acme.com/about-us' });+ // The listed /contact answered 404, so the site was asked.+ expect(checks.contact).toEqual({ ok: true, url: 'https://acme.com/contact' });+ expect(checks.editorial).toEqual({ ok: false, url: null });+ expect(checks.organizationSchema).toEqual({ ok: true, url: 'https://acme.com/' });+ // The brand's authors: the one its pages are by and its default.+ expect(checks.authorPages).toEqual({+ ok: false,+ authors: [+ { name: 'Ada Lovelace', url: 'https://acme.com/authors/ada-lovelace' },+ { name: 'Bob Stone', url: null },+ ],+ });+ expect(score).toBe(3);+ expect(complete).toBe(true);+ expect(fetcher.mock.calls.every(([url]) => url.startsWith('https://acme.com/'))).toBe(true);+ });++ it('takes neither a redirect off the site nor one to the home page, nor passes without authors', async () => {+ const { db } = fakeDb({+ ...tables,+ site_pages: [],+ content_authors: [],+ content_drafts: [],+ content_author_defaults: [],+ });+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/about')+ return { url: 'https://other.example/about', html: html('About us') };+ if (url === 'https://acme.com/about-us')+ return { url: 'https://acme.com/', html: html('About Acme') };+ return { url, html: html('Acme') };+ });+ const { checks, score } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.about.ok).toBe(false);+ expect(checks.authorPages).toEqual({ ok: false, authors: [] });+ expect(score).toBe(0);+ });++ it("marks a read the site didn't answer all of", async () => {+ const { db } = fakeDb(tables);+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') throw new PageFetchError('timeout', 'slow');+ throw new PageFetchError('http_error', '404', { status: 404 });+ });+ const { checks, complete } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.organizationSchema.ok).toBe(false);+ expect(complete).toBe(false);+ });++ it("reads a bot wall as not known yet, not as a page the site doesn't have", async () => {+ const lists = { ...tables, content_authors: [], content_drafts: [] };+ const wall = (status) => new PageFetchError('http_error', String(status), { status });+ for (const blocked of [401, 403, 429]) {+ const { db } = fakeDb(lists);+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') return { url, html: HOME };+ throw wall(blocked);+ });+ const { checks, complete } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.editorial.ok).toBe(false);+ expect(complete).toBe(false);+ }+ // A challenge page that answers 200 is no home page: no Scrape.do to try here.+ const { db } = fakeDb(lists);+ const challenge = { html: html('Just a moment...', '', ORG) };+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') return { url, ...challenge };+ throw wall(404);+ });+ const { checks, complete, scraped } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.organizationSchema.ok).toBe(false);+ expect(complete).toBe(false);+ expect(scraped).toBe(false);+ });++ it('doesn’t take a page that only loads a captcha script for a wall', async () => {+ const { db } = fakeDb({ ...tables, content_authors: [], content_drafts: [] });+ const page = html('Contact – Acme', 'Contact us', '<script src="/recaptcha/api.js"></script>');+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') return { url, html: HOME };+ if (url === 'https://acme.com/contact') return { url, html: page };+ throw new PageFetchError('http_error', '404', { status: 404 });+ });+ const { checks, complete } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.contact.ok).toBe(true);+ expect(complete).toBe(true);+ });++ it('reads a walled home page once through Scrape.do, unrendered', async () => {+ const lists = { ...tables, content_authors: [], content_drafts: [] };+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/')+ throw new PageFetchError('http_error', '403', { status: 403 });+ if (url === 'https://acme.com/contact') return { url, html: html('Contact – Acme') };+ throw new PageFetchError('http_error', '404', { status: 404 });+ });+ let { db } = fakeDb(lists);+ const scraper = vi.fn(async () => ({ ok: true, status: 200, html: HOME }));+ let result = await computeSiteTrust('b1', { db, fetcher, scraper });+ expect(scraper).toHaveBeenCalledTimes(1);+ expect(scraper).toHaveBeenCalledWith('https://acme.com/');+ expect(result.checks.organizationSchema).toEqual({ ok: true, url: 'https://acme.com/' });+ expect(result).toMatchObject({ complete: true, scraped: true });++ // Still walled through it: not known yet.+ ({ db } = fakeDb(lists));+ const walled = vi.fn(async () => ({ ok: false, status: 403, html: '' }));+ result = await computeSiteTrust('b1', { db, fetcher, scraper: walled });+ expect(result.checks.organizationSchema.ok).toBe(false);+ expect(result).toMatchObject({ complete: false, scraped: true });++ // Without Scrape.do (used today): the pass it read stands, else not known yet.+ ({ db } = fakeDb(lists));+ result = await computeSiteTrust('b1', { db, fetcher, keptHome: { ok: true, url: 'x' } });+ expect(result.checks.organizationSchema).toEqual({ ok: true, url: 'https://acme.com/' });+ expect(result).toMatchObject({ complete: true, scraped: false });+ ({ db } = fakeDb(lists));+ result = await computeSiteTrust('b1', { db, fetcher, keptHome: { ok: false, url: 'x' } });+ expect(result).toMatchObject({ complete: false, scraped: false });++ // A slow home page isn't a wall: no credit spent on it.+ ({ db } = fakeDb(lists));+ const slow = vi.fn(async (url) => {+ if (url === 'https://acme.com/') throw new PageFetchError('timeout', 'slow');+ throw new PageFetchError('http_error', '404', { status: 404 });+ });+ scraper.mockClear();+ result = await computeSiteTrust('b1', { db, fetcher: slow, scraper });+ expect(scraper).not.toHaveBeenCalled();+ expect(result.complete).toBe(false);+ });++ it('looks for an author’s page at the usual addresses when the list has none', async () => {+ const { db } = fakeDb({+ ...tables,+ site_pages: [],+ content_authors: [+ { id: 'a1', name: 'Ada Lovelace', same_as: [] },+ { id: 'a2', name: 'Bob Stone', same_as: [] },+ { id: 'a3', name: 'Cal Dee', same_as: [] },+ ],+ content_drafts: [],+ content_author_defaults: [],+ });+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') return { url, html: HOME };+ // WordPress's author archive.+ if (url === 'https://acme.com/author/ada-lovelace')+ return { url, html: html('Ada Lovelace, Author at Acme', 'Author: Ada Lovelace') };+ // The home page again, and a page that doesn't name him.+ if (url === 'https://acme.com/author/bob-stone')+ return { url: 'https://acme.com/', html: HOME };+ if (url === 'https://acme.com/authors/bob-stone') return { url, html: html('Our team') };+ // Off the site.+ if (url === 'https://acme.com/author/cal-dee')+ return { url: 'https://other.example/cal-dee', html: html('Cal Dee') };+ if (url === 'https://acme.com/authors/cal-dee')+ return { url, html: html('Writers', 'Cal Dee') };+ throw new PageFetchError('http_error', '404', { status: 404 });+ });+ const { checks, complete } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.authorPages).toEqual({+ ok: false,+ authors: [+ { name: 'Ada Lovelace', url: 'https://acme.com/author/ada-lovelace' },+ { name: 'Bob Stone', url: null },+ { name: 'Cal Dee', url: 'https://acme.com/authors/cal-dee' },+ ],+ });+ expect(complete).toBe(true);+ // The first address that passes ends the search.+ expect(fetcher).not.toHaveBeenCalledWith(+ 'https://acme.com/authors/ada-lovelace',+ expect.anything(),+ );+ });++ it(`looks for ${MAX_AUTHOR_PROBES} authors' pages at most, and not inside a longer name`, async () => {+ const names = ['Al', 'Bea Moss', 'Cy Ode', 'Di Ray', 'Ed Fox'];+ const { db } = fakeDb({+ ...tables,+ site_pages: [],+ content_authors: names.map((name, i) => ({ id: `a${i}`, name, same_as: [] })),+ content_drafts: [],+ content_author_defaults: [],+ });+ const fetcher = vi.fn(async (url) => {+ if (url === 'https://acme.com/') return { url, html: HOME };+ // "Al" only inside "Also": not his page.+ if (url === 'https://acme.com/author/al') return { url, html: html('Also by our team') };+ throw new PageFetchError('http_error', '404', { status: 404 });+ });+ const { checks } = await computeSiteTrust('b1', { db, fetcher });+ expect(checks.authorPages.authors.every((a) => a.url === null)).toBe(true);+ const authorReads = fetcher.mock.calls.filter(([url]) => /\/authors?\//.test(url));+ expect(authorReads).toHaveLength(MAX_AUTHOR_PROBES * 2);+ });+});++describe('readSiteTrust', () => {+ const kept = (age, complete = true) => ({+ checks: { about: { ok: true, url: 'https://acme.com/about' } },+ score: 1,+ complete,+ checked_at: new Date(NOW - age).toISOString(),+ });+ const base = {+ brands: { organization_id: 'o1' },+ brand_domains: [],+ site_pages: [],+ content_authors: [],+ content_drafts: [],+ content_author_defaults: [],+ };++ it('keeps a read for a day, and reads again when asked, but not within minutes', async () => {+ let { db, writes } = fakeDb({ ...base, site_trust: kept(60_000) });+ const fetcher = vi.fn();+ expect(await readSiteTrust('b1', { db, fetcher, now: NOW })).toEqual({+ checks: { about: { ok: true, url: 'https://acme.com/about' } },+ score: 1,+ complete: true,+ checkedAt: new Date(NOW - 60_000).toISOString(),+ });+ expect((await readSiteTrust('b1', { db, fetcher, now: NOW, refresh: true })).score).toBe(1);+ expect(writes).toEqual([]);++ ({ db, writes } = fakeDb({ ...base, site_trust: kept(RECHECK_MS + 60_000) }));+ expect((await readSiteTrust('b1', { db, fetcher, now: NOW, refresh: true })).score).toBe(0);+ expect(writes).toHaveLength(1);++ ({ db, writes } = fakeDb({ ...base, site_trust: kept(SITE_TRUST_TTL_MS + 60_000) }));+ const fresh = await readSiteTrust('b1', { db, fetcher, now: NOW });+ expect(fresh.checkedAt).toBe(new Date(NOW).toISOString());+ expect(writes[0]).toMatchObject({+ table: 'site_trust',+ row: { brand_id: 'b1', score: 0, complete: true },+ });+ });++ it('reads a partial read again after a few minutes, and says it is partial', async () => {+ let { db, writes } = fakeDb({ ...base, site_trust: kept(60_000, false) });+ expect((await readSiteTrust('b1', { db, fetcher: vi.fn(), now: NOW })).complete).toBe(false);+ ({ db, writes } = fakeDb({ ...base, site_trust: kept(RECHECK_MS + 60_000, false) }));+ await readSiteTrust('b1', { db, fetcher: vi.fn(), now: NOW });+ expect(writes).toHaveLength(1);+ });++ it('spends one Scrape.do read a day on a walled home page, however often it is asked', async () => {+ const tables = {+ ...base,+ brand_domains: [{ domain: 'acme.com', is_primary: true }],+ };+ const fetcher = vi.fn(async () => {+ throw new PageFetchError('http_error', '403', { status: 403 });+ });+ const scraper = vi.fn(async () => ({ ok: true, status: 200, html: html('Acme', 'Acme', ORG) }));+ let { db, writes } = fakeDb(tables);+ let read = await readSiteTrust('b-walled', { db, fetcher, scraper, now: NOW });+ expect(scraper).toHaveBeenCalledTimes(1);+ expect(read.checks.organizationSchema.ok).toBe(true);+ // The probes met the wall too: read again soon.+ expect(read.complete).toBe(false);+ expect(writes[0].row.scraped_at).toBe(new Date(NOW).toISOString());++ // Asked again later that day: no second credit, and the pass stands.+ let later = NOW + RECHECK_MS + 60_000;+ ({ db, writes } = fakeDb({ ...tables, site_trust: writes[0].row }));+ read = await readSiteTrust('b-walled', { db, fetcher, scraper, now: later, refresh: true });+ expect(scraper).toHaveBeenCalledTimes(1);+ expect(read.checks.organizationSchema.ok).toBe(true);+ expect(writes[0].row.scraped_at).toBe(new Date(NOW).toISOString());++ // A day on: one more.+ later = NOW + SITE_TRUST_TTL_MS + 60_000;+ ({ db, writes } = fakeDb({ ...tables, site_trust: writes[0].row }));+ await readSiteTrust('b-walled', { db, fetcher, scraper, now: later });+ expect(scraper).toHaveBeenCalledTimes(2);+ expect(writes[0].row.scraped_at).toBe(new Date(later).toISOString());+ });++ it('reads once for two asks at the same time', async () => {+ const { db, writes } = fakeDb(base);+ const [a, b] = await Promise.all([+ readSiteTrust('b-twice', { db, fetcher: vi.fn(), now: NOW }),+ readSiteTrust('b-twice', { db, fetcher: vi.fn(), now: NOW }),+ ]);+ expect(writes).toHaveLength(1);+ expect(a).toEqual(b);+ });+});diff --git a/server/src/routes/audits-settings.test.js b/server/src/routes/audits-settings.test.jsindex 6b22a04ce..c98173eff 100644--- a/server/src/routes/audits-settings.test.js+++ b/server/src/routes/audits-settings.test.js@@ -10,6 +10,7 @@ import express from 'express';let tables;let writes;let quota;+const readSiteTrust = vi.fn();let server;let baseUrl;@@ -81,6 +82,7 @@ async function startApp() {brandFactsFor: vi.fn(),generateRecommendations: vi.fn(),}));+ vi.doMock('../lib/site-trust.js', () => ({ readSiteTrust }));vi.doMock('../lib/logger.js', () => {const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };return { default: logger, logger };@@ -106,6 +108,12 @@ const otherBrand = '44444444-4444-4444-4444-444444444444';beforeEach(async () => {writes = [];+ readSiteTrust.mockReset().mockResolvedValue({+ checks: { about: { ok: true, url: 'https://acme.com/about' } },+ score: 1,+ complete: false,+ checkedAt: '2026-10-23T12:00:00.000Z',+ });quota = {used: 10,running: 0,@@ -279,3 +287,27 @@ describe('GET /api/audits/auto-runs/latest', () => {expect((await res.json()).run).toBeNull();});});++describe('GET /api/audits/site-trust', () => {+ it("answers with the brand's site trust, and reads it again when asked", async () => {+ let res = await fetch(`${baseUrl}/site-trust?brandId=${brandId}`);+ expect(res.status).toBe(200);+ // complete: whether the site answered all of it; false, the card says it checks again soon.+ await expect(res.json()).resolves.toEqual({+ success: true,+ checks: { about: { ok: true, url: 'https://acme.com/about' } },+ score: 1,+ complete: false,+ checkedAt: '2026-10-23T12:00:00.000Z',+ });+ expect(readSiteTrust).toHaveBeenLastCalledWith(brandId, { refresh: false });+ res = await fetch(`${baseUrl}/site-trust?brandId=${brandId}&refresh=1`);+ expect(readSiteTrust).toHaveBeenLastCalledWith(brandId, { refresh: true });+ });++ it('wants a brand id', async () => {+ const res = await fetch(`${baseUrl}/site-trust?brandId=nope`);+ expect(res.status).toBe(400);+ expect(readSiteTrust).not.toHaveBeenCalled();+ });+});diff --git a/server/src/routes/audits.js b/server/src/routes/audits.jsindex 1b78a94e6..b67a5fd67 100644--- a/server/src/routes/audits.js+++ b/server/src/routes/audits.js@@ -46,6 +46,7 @@ import {} from '../lib/audit/opportunities.js';import { AUTO_AUDIT } from '../lib/audit/auto-audit.js';import { rankOpportunities } from '../lib/audit/priority.js';+import { readSiteTrust } from '../lib/site-trust.js';import { buildTemplates, templateOf, templateOpportunities } from '../lib/audit/templates.js';import { ENGINE_THRESHOLDS } from '../config/action-engine.js';import { runSignals } from '../lib/audit/engine.js';@@ -1162,6 +1163,23 @@ router.get('/auto-runs/latest', async (req, res) => {}});+// GET /api/audits/site-trust?brandId=…&refresh=1 — the brand's five site+// trust checks (Content Editor Quality, CE-043): the day's read, or a new+// one when asked again (site-trust.js). Registered before /:id.+router.get('/site-trust', async (req, res) => {+ const parsed = z.guid().safeParse(req.query.brandId);+ if (!parsed.success) {+ return res.status(400).json({ success: false, message: 'brandId is required' });+ }+ try {+ await assertBrandAccess(parsed.data, req.user?.id);+ const trust = await readSiteTrust(parsed.data, { refresh: req.query.refresh === '1' });+ return res.json({ success: true, ...trust });+ } catch (err) {+ return sendAuditError(req, res, err, 'site trust read failed');+ }+});+// GET /api/audits/:id — fetch one stored audit with its signalsrouter.get('/:id', async (req, res) => {const userId = req.user?.id;diff --git a/supabase/migrations/00259_site_trust.sql b/supabase/migrations/00259_site_trust.sqlnew file mode 100644index 000000000..aa659b6da--- /dev/null+++ b/supabase/migrations/00259_site_trust.sql@@ -0,0 +1,51 @@+-- ============================================================================+-- 00259: a brand's site trust (Content Editor Quality, CE-043)+-- ============================================================================+-- Site trust (Quality spec §8): five site-level checks Site Audit shows as+-- "Site trust {n}/5", and Trust gives 5 points on every page from:+--+-- * about: an About page;+-- * contact: a Contact page;+-- * editorial: an editorial policy (or "How we review") page;+-- * organizationSchema: Organization schema on the home page;+-- * authorPages: a page for each author in Settings › Authors.+--+-- The server reads them from the brand's page list (site_pages), its home+-- page and its authors, at most once a day or when someone asks again; this+-- is the last read. complete is false when the site didn't answer all of it+-- (slow, unreachable, a server error, a bot wall): such a read is read again+-- after a few minutes. scraped_at is when the home page last went through+-- Scrape.do, past a bot wall: at most once a day per brand, one credit.+-- Members read it; only the server writes it.++create table if not exists public.site_trust (+ brand_id uuid primary key references public.brands(id) on delete cascade,+ checks jsonb not null,+ score smallint not null check (score between 0 and 5),+ complete boolean not null default true,+ checked_at timestamptz not null default now(),+ scraped_at timestamptz+);++-- For a database that ran this file before scraped_at was added to it.+alter table public.site_trust add column if not exists scraped_at timestamptz;++alter table public.site_trust enable row level security;+revoke all on table public.site_trust from anon, authenticated;+grant select on table public.site_trust to authenticated;+grant all on table public.site_trust to service_role;++drop policy if exists "site_trust: member select" on public.site_trust;+create policy "site_trust: member select"+ on public.site_trust+ for select+ using (+ brand_id in (+ select b.id from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++comment on table public.site_trust is+ 'Content Editor Quality (CE-043): a brand''s five site trust checks (About, Contact, editorial policy, Organization schema on the home page, author pages) and how many pass. Written by the server, read by members.';diff --git a/supabase/schema.sql b/supabase/schema.sqlindex e00e87e37..2c55fcef8 100644--- a/supabase/schema.sql+++ b/supabase/schema.sql@@ -37924,3 +37924,58 @@ grant all on table public.content_link_checks to service_role;comment on table public.content_link_checks is'Content Editor Quality (Trust): each linked page''s last read — answers or not, and its own date. Server only; read at most once a day, kept a week.';+-- ─────────────────────────────────────────────────────────────────────────+-- migrations/00259_site_trust.sql+-- ─────────────────────────────────────────────────────────────────────────+-- ============================================================================+-- 00259: a brand's site trust (Content Editor Quality, CE-043)+-- ============================================================================+-- Site trust (Quality spec §8): five site-level checks Site Audit shows as+-- "Site trust {n}/5", and Trust gives 5 points on every page from:+--+-- * about: an About page;+-- * contact: a Contact page;+-- * editorial: an editorial policy (or "How we review") page;+-- * organizationSchema: Organization schema on the home page;+-- * authorPages: a page for each author in Settings › Authors.+--+-- The server reads them from the brand's page list (site_pages), its home+-- page and its authors, at most once a day or when someone asks again; this+-- is the last read. complete is false when the site didn't answer all of it+-- (slow, unreachable, a server error, a bot wall): such a read is read again+-- after a few minutes. scraped_at is when the home page last went through+-- Scrape.do, past a bot wall: at most once a day per brand, one credit.+-- Members read it; only the server writes it.++create table if not exists public.site_trust (+ brand_id uuid primary key references public.brands(id) on delete cascade,+ checks jsonb not null,+ score smallint not null check (score between 0 and 5),+ complete boolean not null default true,+ checked_at timestamptz not null default now(),+ scraped_at timestamptz+);++-- For a database that ran this file before scraped_at was added to it.+alter table public.site_trust add column if not exists scraped_at timestamptz;++alter table public.site_trust enable row level security;+revoke all on table public.site_trust from anon, authenticated;+grant select on table public.site_trust to authenticated;+grant all on table public.site_trust to service_role;++drop policy if exists "site_trust: member select" on public.site_trust;+create policy "site_trust: member select"+ on public.site_trust+ for select+ using (+ brand_id in (+ select b.id from public.brands b+ join public.profiles p on p.organization_id = b.organization_id+ where p.id = auth.uid()+ )+ );++comment on table public.site_trust is+ 'Content Editor Quality (CE-043): a brand''s five site trust checks (About, Contact, editorial policy, Organization schema on the home page, author pages) and how many pass. Written by the server, read by members.';+